openstack/trove
Code Issues Proposed changes

Adding support for encrypted backups.

Browse Source 
We'd like to enable the backups that are taken and stored as files in Swift, to
be encrypted. We'll add some new config flags, to mark whether or not
encryption is enabled. At this point, the encryption key will also be stored as
a config value. When configured to encrypt, the whole process will be streamed
to/from Swift, doing backup -> zip -> encrypt -> Swift and Swift -> decrypt ->
unzip -> restore all during streaming, so we don't use up more disk space than
needed.

Blueprint: encrypted-backups
Change-Id: I05447306f4249bfd8e02de7b008ebda3387b2fbd
This commit is contained in:
Dror Kagan 2013-05-28 16:28:54 -07:00 committed by Nikhil Manchanda
parent 35c87da0a2
commit 4eef8d9a70
8 changed files with 173 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
etc/reddwarf/reddwarf-guestagent.conf.sample
View File

@ -72,6 +72,8 @@ storage_strategy = SwiftStorage
storage_namespace = reddwarf.guestagent.strategies.storage.swift
backup_swift_container = database_backups
backup_use_gzip_compression = True
backup_use_openssl_encryption = True
backup_aes_cbc_key = "default_aes_cbc_key"
backup_use_snet = False
backup_chunk_size = 65536
backup_segment_max_size = 2147483648

4
reddwarf/common/cfg.py
View File

@ -140,6 +140,10 @@ common_opts = [
cfg.StrOpt('backup_swift_container', default='database_backups'),
cfg.BoolOpt('backup_use_gzip_compression', default=True,
help='Compress backups using gzip.'),
cfg.BoolOpt('backup_use_openssl_encryption', default=True,
help='Encrypt backups using openssl.'),
cfg.StrOpt('backup_aes_cbc_key', default='default_aes_cbc_key',
help='default openssl aes_cbc key.'),
cfg.BoolOpt('backup_use_snet', default=False,
help='Send backup files over snet.'),
cfg.IntOpt('backup_chunk_size', default=2 ** 16,

14
reddwarf/guestagent/strategies/backup/base.py
View File

@ -30,6 +30,9 @@ CHUNK_SIZE = CONF.backup_chunk_size
MAX_FILE_SIZE = CONF.backup_segment_max_size
BACKUP_CONTAINER = CONF.backup_swift_container
BACKUP_USE_GZIP = CONF.backup_use_gzip_compression
BACKUP_USE_OPENSSL = CONF.backup_use_openssl_encryption
BACKUP_ENCRYPT_KEY = CONF.backup_aes_cbc_key
LOG = logging.getLogger(__name__)
@ -49,6 +52,8 @@ class BackupRunner(Strategy):
# The actual system call to run the backup
cmd = None
is_zipped = BACKUP_USE_GZIP
is_encrypted = BACKUP_USE_OPENSSL
encrypt_key = BACKUP_ENCRYPT_KEY
def __init__(self, filename, **kwargs):
self.filename = filename
@ -119,6 +124,15 @@ class BackupRunner(Strategy):
def zip_manifest(self):
return '.gz' if self.is_zipped else ''
@property
def encrypt_cmd(self):
return (' | openssl enc -aes-256-cbc -salt -pass pass:%s' %
self.encrypt_key) if self.is_encrypted else ''
@property
def encrypt_manifest(self):
return '.enc' if self.is_encrypted else ''
def read(self, chunk_size):
"""Wrap self.process.stdout.read to allow for segmentation."""
if self.end_of_segment:

12
reddwarf/guestagent/strategies/backup/impl.py
View File

@ -32,12 +32,12 @@ class MySQLDump(base.BackupRunner):
' --opt'\
' --password=%(password)s'\
' -u %(user)s'
return cmd + self.zip_cmd
return cmd + self.zip_cmd + self.encrypt_cmd
@property
def manifest(self):
manifest = '%s' + self.zip_manifest
return manifest % self.filename
manifest = '%s' % self.filename
return manifest + self.zip_manifest + self.encrypt_manifest
class InnoBackupEx(base.BackupRunner):
@ -49,9 +49,9 @@ class InnoBackupEx(base.BackupRunner):
cmd = 'sudo innobackupex'\
' --stream=xbstream'\
' /var/lib/mysql 2>/tmp/innobackupex.log'
return cmd + self.zip_cmd
return cmd + self.zip_cmd + self.encrypt_cmd
@property
def manifest(self):
manifest = '%s.xbstream' + self.zip_manifest
return manifest % self.filename
manifest = '%s.xbstream' % self.filename
return manifest + self.zip_manifest + self.encrypt_manifest

28
reddwarf/guestagent/strategies/restore/base.py
View File

@ -27,6 +27,9 @@ import glob
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
CHUNK_SIZE = CONF.backup_chunk_size
BACKUP_USE_GZIP = CONF.backup_use_gzip_compression
BACKUP_USE_OPENSSL = CONF.backup_use_openssl_encryption
BACKUP_DECRYPT_KEY = CONF.backup_aes_cbc_key
RESET_ROOT_RETRY_TIMEOUT = 100
RESET_ROOT_SLEEP_INTERVAL = 10
RESET_ROOT_MYSQL_COMMAND = """
@ -65,13 +68,20 @@ class RestoreRunner(Strategy):
# The backup format type
restore_type = None
# Decryption Parameters
is_zipped = BACKUP_USE_GZIP
is_encrypted = BACKUP_USE_OPENSSL
decrypt_key = BACKUP_DECRYPT_KEY
def __init__(self, restore_stream, **kwargs):
self.restore_stream = restore_stream
self.restore_location = kwargs.get('restore_location',
'/var/lib/mysql')
self.restore_cmd = self.restore_cmd % kwargs
self.prepare_cmd = self.prepare_cmd % kwargs \
if hasattr(self, 'prepare_cmd') else None
self.restore_cmd = (self.decrypt_cmd +
self.unzip_cmd +
(self.base_restore_cmd % kwargs))
self.prepare_cmd = self.base_prepare_cmd % kwargs \
if hasattr(self, 'base_prepare_cmd') else None
super(RestoreRunner, self).__init__()
def __enter__(self):
@ -164,3 +174,15 @@ class RestoreRunner(Strategy):
filelist = glob.glob(self.restore_location + "/ib_logfile*")
for f in filelist:
os.unlink(f)
@property
def decrypt_cmd(self):
if self.is_encrypted:
return ('openssl enc -d -aes-256-cbc -salt -pass pass:%s | '
% self.decrypt_key)
else:
return ''
@property
def unzip_cmd(self):
return 'gzip -d -c | ' if self.is_zipped else ''

16
reddwarf/guestagent/strategies/restore/impl.py
View File

@ -25,10 +25,9 @@ LOG = logging.getLogger(__name__)
class MySQLDump(base.RestoreRunner):
""" Implementation of Restore Strategy for MySQLDump """
__strategy_name__ = 'mysqldump'
is_zipped = True
restore_cmd = ('mysql '
'--password=%(password)s '
'-u %(user)s')
base_restore_cmd = ('mysql '
'--password=%(password)s '
'-u %(user)s')
def _pre_restore(self):
pass
@ -40,11 +39,10 @@ class MySQLDump(base.RestoreRunner):
class InnoBackupEx(base.RestoreRunner):
""" Implementation of Restore Strategy for InnoBackupEx """
__strategy_name__ = 'innobackupex'
is_zipped = True
restore_cmd = 'sudo xbstream -x -C %(restore_location)s'
prepare_cmd = ('sudo innobackupex --apply-log %(restore_location)s '
'--defaults-file=%(restore_location)s/backup-my.cnf '
'--ibbackup xtrabackup 2>/tmp/innoprepare.log')
base_restore_cmd = 'sudo xbstream -x -C %(restore_location)s'
base_prepare_cmd = ('sudo innobackupex --apply-log %(restore_location)s'
' --defaults-file=%(restore_location)s/backup-my.cnf'
' --ibbackup xtrabackup 2>/tmp/innoprepare.log')
def _pre_restore(self):
app = dbaas.MySqlApp(dbaas.MySqlAppStatus.get())

6
reddwarf/guestagent/strategies/storage/swift.py
View File

@ -21,7 +21,6 @@ from reddwarf.common import utils
from eventlet.green import subprocess
import zlib
UNZIPPER = zlib.decompressobj(16 + zlib.MAX_WBITS)
LOG = logging.getLogger(__name__)
@ -107,7 +106,6 @@ class SwiftDownloadStream(object):
def __init__(self, **kwargs):
self.process = None
self.pid = None
self.is_zipped = kwargs.get('is_zipped', False)
self.cmd = self.cmd % kwargs
def __enter__(self):
@ -128,9 +126,7 @@ class SwiftDownloadStream(object):
pass
def read(self, *args, **kwargs):
if not self.is_zipped:
return self.process.stdout.read(*args, **kwargs)
return UNZIPPER.decompress(self.process.stdout.read(*args, **kwargs))
return self.process.stdout.read(*args, **kwargs)
def run(self):
self.process = subprocess.Popen(self.cmd, shell=True,

114
reddwarf/tests/unittests/guestagent/test_backups.py Normal file
View File

@ -0,0 +1,114 @@
# Copyright 2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License
import reddwarf.guestagent.strategies.backup.base as backupBase
import reddwarf.guestagent.strategies.restore.base as restoreBase
import testtools
from reddwarf.common import utils
BACKUP_XTRA_CLS = "reddwarf.guestagent.strategies.backup.impl.InnoBackupEx"
RESTORE_XTRA_CLS = "reddwarf.guestagent.strategies.restore.impl.InnoBackupEx"
BACKUP_SQLDUMP_CLS = "reddwarf.guestagent.strategies.backup.impl.MySQLDump"
RESTORE_SQLDUMP_CLS = "reddwarf.guestagent.strategies.restore.impl.MySQLDump"
PIPE = " | "
ZIP = "gzip"
UNZIP = "gzip -d -c"
ENCRYPT = "openssl enc -aes-256-cbc -salt -pass pass:default_aes_cbc_key"
DECRYPT = "openssl enc -d -aes-256-cbc -salt -pass pass:default_aes_cbc_key"
XTRA_BACKUP = "sudo innobackupex --stream=xbstream /var/lib/mysql 2>/" \
"tmp/innobackupex.log"
SQLDUMP_BACKUP = "/usr/bin/mysqldump --all-databases --opt " \
"--password=password -u user"
XTRA_RESTORE = "sudo xbstream -x -C /var/lib/mysql"
SQLDUMP_RESTORE = "mysql --password=password -u user"
PREPARE = "sudo innobackupex --apply-log /var/lib/mysql " \
"--defaults-file=/var/lib/mysql/backup-my.cnf " \
"--ibbackup xtrabackup 2>/tmp/innoprepare.log"
CRYPTO_KEY = "default_aes_cbc_key"
class GuestAgentBackupTest(testtools.TestCase):
def test_backup_decrypted_xtrabackup_command(self):
backupBase.BackupRunner.is_zipped = True
backupBase.BackupRunner.is_encrypted = False
RunnerClass = utils.import_class(BACKUP_XTRA_CLS)
bkup = RunnerClass(12345, user="user", password="password")
self.assertEqual(bkup.command, XTRA_BACKUP + PIPE + ZIP)
self.assertEqual(bkup.manifest, "12345.xbstream.gz")
def test_backup_encrypted_xtrabackup_command(self):
backupBase.BackupRunner.is_zipped = True
backupBase.BackupRunner.is_encrypted = True
backupBase.BackupRunner.encrypt_key = CRYPTO_KEY
RunnerClass = utils.import_class(BACKUP_XTRA_CLS)
bkup = RunnerClass(12345, user="user", password="password")
self.assertEqual(bkup.command,
XTRA_BACKUP + PIPE + ZIP + PIPE + ENCRYPT)
self.assertEqual(bkup.manifest, "12345.xbstream.gz.enc")
def test_backup_decrypted_mysqldump_command(self):
backupBase.BackupRunner.is_zipped = True
backupBase.BackupRunner.is_encrypted = False
RunnerClass = utils.import_class(BACKUP_SQLDUMP_CLS)
bkup = RunnerClass(12345, user="user", password="password")
self.assertEqual(bkup.command, SQLDUMP_BACKUP + PIPE + ZIP)
self.assertEqual(bkup.manifest, "12345.gz")
def test_backup_encrypted_mysqldump_command(self):
backupBase.BackupRunner.is_zipped = True
backupBase.BackupRunner.is_encrypted = True
backupBase.BackupRunner.encrypt_key = CRYPTO_KEY
RunnerClass = utils.import_class(BACKUP_SQLDUMP_CLS)
bkup = RunnerClass(12345, user="user", password="password")
self.assertEqual(bkup.command,
SQLDUMP_BACKUP + PIPE + ZIP + PIPE + ENCRYPT)
self.assertEqual(bkup.manifest, "12345.gz.enc")
def test_restore_decrypted_xtrabackup_command(self):
restoreBase.RestoreRunner.is_zipped = True
restoreBase.RestoreRunner.is_encrypted = False
RunnerClass = utils.import_class(RESTORE_XTRA_CLS)
restr = RunnerClass(None, restore_location="/var/lib/mysql")
self.assertEqual(restr.restore_cmd, UNZIP + PIPE + XTRA_RESTORE)
self.assertEqual(restr.prepare_cmd, PREPARE)
def test_restore_encrypted_xtrabackup_command(self):
restoreBase.RestoreRunner.is_zipped = True
restoreBase.RestoreRunner.is_encrypted = True
restoreBase.RestoreRunner.decrypt_key = CRYPTO_KEY
RunnerClass = utils.import_class(RESTORE_XTRA_CLS)
restr = RunnerClass(None, restore_location="/var/lib/mysql")
self.assertEqual(restr.restore_cmd,
DECRYPT + PIPE + UNZIP + PIPE + XTRA_RESTORE)
self.assertEqual(restr.prepare_cmd, PREPARE)
def test_restore_decrypted_mysqldump_command(self):
restoreBase.RestoreRunner.is_zipped = True
restoreBase.RestoreRunner.is_encrypted = False
RunnerClass = utils.import_class(RESTORE_SQLDUMP_CLS)
restr = RunnerClass(None, restore_location="/var/lib/mysql",
user="user", password="password")
self.assertEqual(restr.restore_cmd, UNZIP + PIPE + SQLDUMP_RESTORE)
self.assertIsNone(restr.prepare_cmd)
def test_restore_encrypted_mysqldump_command(self):
restoreBase.RestoreRunner.is_zipped = True
restoreBase.RestoreRunner.is_encrypted = True
restoreBase.RestoreRunner.decrypt_key = CRYPTO_KEY
RunnerClass = utils.import_class(RESTORE_SQLDUMP_CLS)
restr = RunnerClass(None, restore_location="/var/lib/mysql",
user="user", password="password")
self.assertEqual(restr.restore_cmd,
DECRYPT + PIPE + UNZIP + PIPE + SQLDUMP_RESTORE)
self.assertIsNone(restr.prepare_cmd)