Merge "Adding support for encrypted backups."

etc/reddwarf/reddwarf-guestagent.conf.sample

@@ -72,6 +72,8 @@ storage_strategy = SwiftStorage
 storage_namespace = reddwarf.guestagent.strategies.storage.swift
 backup_swift_container = database_backups
 backup_use_gzip_compression = True
+backup_use_openssl_encryption = True
+backup_aes_cbc_key = "default_aes_cbc_key"
 backup_use_snet = False
 backup_chunk_size = 65536
 backup_segment_max_size = 2147483648

reddwarf/common/cfg.py

@@ -140,6 +140,10 @@ common_opts = [
     cfg.StrOpt('backup_swift_container', default='database_backups'),
     cfg.BoolOpt('backup_use_gzip_compression', default=True,
                 help='Compress backups using gzip.'),
+    cfg.BoolOpt('backup_use_openssl_encryption', default=True,
+                help='Encrypt backups using openssl.'),
+    cfg.StrOpt('backup_aes_cbc_key', default='default_aes_cbc_key',
+               help='default openssl aes_cbc key.'),
     cfg.BoolOpt('backup_use_snet', default=False,
                 help='Send backup files over snet.'),
     cfg.IntOpt('backup_chunk_size', default=2 ** 16,

reddwarf/guestagent/strategies/backup/base.py

@@ -30,6 +30,9 @@ CHUNK_SIZE = CONF.backup_chunk_size
 MAX_FILE_SIZE = CONF.backup_segment_max_size
 BACKUP_CONTAINER = CONF.backup_swift_container
 BACKUP_USE_GZIP = CONF.backup_use_gzip_compression
+BACKUP_USE_OPENSSL = CONF.backup_use_openssl_encryption
+BACKUP_ENCRYPT_KEY = CONF.backup_aes_cbc_key
+
 LOG = logging.getLogger(__name__)
 
 
@@ -49,6 +52,8 @@ class BackupRunner(Strategy):
     # The actual system call to run the backup
     cmd = None
     is_zipped = BACKUP_USE_GZIP
+    is_encrypted = BACKUP_USE_OPENSSL
+    encrypt_key = BACKUP_ENCRYPT_KEY
 
     def __init__(self, filename, **kwargs):
         self.filename = filename
@@ -119,6 +124,15 @@ class BackupRunner(Strategy):
     def zip_manifest(self):
         return '.gz' if self.is_zipped else ''
 
+    @property
+    def encrypt_cmd(self):
+        return (' | openssl enc -aes-256-cbc -salt -pass pass:%s' %
+                self.encrypt_key) if self.is_encrypted else ''
+
+    @property
+    def encrypt_manifest(self):
+        return '.enc' if self.is_encrypted else ''
+
     def read(self, chunk_size):
         """Wrap self.process.stdout.read to allow for segmentation."""
         if self.end_of_segment:

reddwarf/guestagent/strategies/backup/impl.py

@@ -32,12 +32,12 @@ class MySQLDump(base.BackupRunner):
             ' --opt'\
             ' --password=%(password)s'\
             ' -u %(user)s'
-        return cmd + self.zip_cmd
+        return cmd + self.zip_cmd + self.encrypt_cmd
 
     @property
     def manifest(self):
-        manifest = '%s' + self.zip_manifest
+        manifest = '%s' % self.filename
-        return manifest % self.filename
+        return manifest + self.zip_manifest + self.encrypt_manifest
 
 
 class InnoBackupEx(base.BackupRunner):
@@ -49,9 +49,9 @@ class InnoBackupEx(base.BackupRunner):
         cmd = 'sudo innobackupex'\
             ' --stream=xbstream'\
             ' /var/lib/mysql 2>/tmp/innobackupex.log'
-        return cmd + self.zip_cmd
+        return cmd + self.zip_cmd + self.encrypt_cmd
 
     @property
     def manifest(self):
-        manifest = '%s.xbstream' + self.zip_manifest
+        manifest = '%s.xbstream' % self.filename
-        return manifest % self.filename
+        return manifest + self.zip_manifest + self.encrypt_manifest

reddwarf/guestagent/strategies/restore/base.py

@@ -27,6 +27,9 @@ import glob
 LOG = logging.getLogger(__name__)
 CONF = cfg.CONF
 CHUNK_SIZE = CONF.backup_chunk_size
+BACKUP_USE_GZIP = CONF.backup_use_gzip_compression
+BACKUP_USE_OPENSSL = CONF.backup_use_openssl_encryption
+BACKUP_DECRYPT_KEY = CONF.backup_aes_cbc_key
 RESET_ROOT_RETRY_TIMEOUT = 100
 RESET_ROOT_SLEEP_INTERVAL = 10
 RESET_ROOT_MYSQL_COMMAND = """
@@ -78,13 +81,20 @@ class RestoreRunner(Strategy):
     # The backup format type
     restore_type = None
 
+    # Decryption Parameters
+    is_zipped = BACKUP_USE_GZIP
+    is_encrypted = BACKUP_USE_OPENSSL
+    decrypt_key = BACKUP_DECRYPT_KEY
+
     def __init__(self, restore_stream, **kwargs):
         self.restore_stream = restore_stream
         self.restore_location = kwargs.get('restore_location',
                                            '/var/lib/mysql')
-        self.restore_cmd = self.restore_cmd % kwargs
+        self.restore_cmd = (self.decrypt_cmd +
-        self.prepare_cmd = self.prepare_cmd % kwargs \
+                            self.unzip_cmd +
-            if hasattr(self, 'prepare_cmd') else None
+                            (self.base_restore_cmd % kwargs))
+        self.prepare_cmd = self.base_prepare_cmd % kwargs \
+            if hasattr(self, 'base_prepare_cmd') else None
         super(RestoreRunner, self).__init__()
 
     def __enter__(self):
@@ -175,3 +185,15 @@ class RestoreRunner(Strategy):
         filelist = glob.glob(self.restore_location + "/ib_logfile*")
         for f in filelist:
             os.unlink(f)
+
+    @property
+    def decrypt_cmd(self):
+        if self.is_encrypted:
+            return ('openssl enc -d -aes-256-cbc -salt -pass pass:%s | '
+                    % self.decrypt_key)
+        else:
+            return ''
+
+    @property
+    def unzip_cmd(self):
+        return 'gzip -d -c | ' if self.is_zipped else ''

reddwarf/guestagent/strategies/restore/impl.py

@@ -25,10 +25,9 @@ LOG = logging.getLogger(__name__)
 class MySQLDump(base.RestoreRunner):
     """ Implementation of Restore Strategy for MySQLDump """
     __strategy_name__ = 'mysqldump'
-    is_zipped = True
+    base_restore_cmd = ('mysql '
-    restore_cmd = ('mysql '
+                        '--password=%(password)s '
-                   '--password=%(password)s '
+                        '-u %(user)s')
-                   '-u %(user)s')
 
     def _pre_restore(self):
         pass
@@ -40,11 +39,10 @@ class MySQLDump(base.RestoreRunner):
 class InnoBackupEx(base.RestoreRunner):
     """ Implementation of Restore Strategy for InnoBackupEx """
     __strategy_name__ = 'innobackupex'
-    is_zipped = True
+    base_restore_cmd = 'sudo xbstream -x -C %(restore_location)s'
-    restore_cmd = 'sudo xbstream -x -C %(restore_location)s'
+    base_prepare_cmd = ('sudo innobackupex --apply-log %(restore_location)s'
-    prepare_cmd = ('sudo innobackupex --apply-log %(restore_location)s '
+                        ' --defaults-file=%(restore_location)s/backup-my.cnf'
-                   '--defaults-file=%(restore_location)s/backup-my.cnf '
+                        ' --ibbackup xtrabackup 2>/tmp/innoprepare.log')
-                   '--ibbackup xtrabackup 2>/tmp/innoprepare.log')
 
     def _pre_restore(self):
         app = dbaas.MySqlApp(dbaas.MySqlAppStatus.get())

reddwarf/guestagent/strategies/storage/swift.py

@@ -21,7 +21,6 @@ from reddwarf.common import utils
 from eventlet.green import subprocess
 import zlib
 
-UNZIPPER = zlib.decompressobj(16 + zlib.MAX_WBITS)
 LOG = logging.getLogger(__name__)
 
 
@@ -107,7 +106,6 @@ class SwiftDownloadStream(object):
     def __init__(self, **kwargs):
         self.process = None
         self.pid = None
-        self.is_zipped = kwargs.get('is_zipped', False)
         self.cmd = self.cmd % kwargs
 
     def __enter__(self):
@@ -128,9 +126,7 @@ class SwiftDownloadStream(object):
             pass
 
     def read(self, *args, **kwargs):
-        if not self.is_zipped:
+        return self.process.stdout.read(*args, **kwargs)
-            return self.process.stdout.read(*args, **kwargs)
-        return UNZIPPER.decompress(self.process.stdout.read(*args, **kwargs))
 
     def run(self):
         self.process = subprocess.Popen(self.cmd, shell=True,

reddwarf/tests/unittests/guestagent/test_backups.py

@@ -0,0 +1,114 @@
+#    Copyright 2012 OpenStack Foundation
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License
+
+import reddwarf.guestagent.strategies.backup.base as backupBase
+import reddwarf.guestagent.strategies.restore.base as restoreBase
+import testtools
+from reddwarf.common import utils
+
+BACKUP_XTRA_CLS = "reddwarf.guestagent.strategies.backup.impl.InnoBackupEx"
+RESTORE_XTRA_CLS = "reddwarf.guestagent.strategies.restore.impl.InnoBackupEx"
+BACKUP_SQLDUMP_CLS = "reddwarf.guestagent.strategies.backup.impl.MySQLDump"
+RESTORE_SQLDUMP_CLS = "reddwarf.guestagent.strategies.restore.impl.MySQLDump"
+PIPE = " | "
+ZIP = "gzip"
+UNZIP = "gzip -d -c"
+ENCRYPT = "openssl enc -aes-256-cbc -salt -pass pass:default_aes_cbc_key"
+DECRYPT = "openssl enc -d -aes-256-cbc -salt -pass pass:default_aes_cbc_key"
+XTRA_BACKUP = "sudo innobackupex --stream=xbstream /var/lib/mysql 2>/" \
+              "tmp/innobackupex.log"
+SQLDUMP_BACKUP = "/usr/bin/mysqldump --all-databases --opt " \
+                 "--password=password -u user"
+XTRA_RESTORE = "sudo xbstream -x -C /var/lib/mysql"
+SQLDUMP_RESTORE = "mysql --password=password -u user"
+PREPARE = "sudo innobackupex --apply-log /var/lib/mysql " \
+          "--defaults-file=/var/lib/mysql/backup-my.cnf " \
+          "--ibbackup xtrabackup 2>/tmp/innoprepare.log"
+CRYPTO_KEY = "default_aes_cbc_key"
+
+
+class GuestAgentBackupTest(testtools.TestCase):
+    def test_backup_decrypted_xtrabackup_command(self):
+        backupBase.BackupRunner.is_zipped = True
+        backupBase.BackupRunner.is_encrypted = False
+        RunnerClass = utils.import_class(BACKUP_XTRA_CLS)
+        bkup = RunnerClass(12345, user="user", password="password")
+        self.assertEqual(bkup.command, XTRA_BACKUP + PIPE + ZIP)
+        self.assertEqual(bkup.manifest, "12345.xbstream.gz")
+
+    def test_backup_encrypted_xtrabackup_command(self):
+        backupBase.BackupRunner.is_zipped = True
+        backupBase.BackupRunner.is_encrypted = True
+        backupBase.BackupRunner.encrypt_key = CRYPTO_KEY
+        RunnerClass = utils.import_class(BACKUP_XTRA_CLS)
+        bkup = RunnerClass(12345, user="user", password="password")
+        self.assertEqual(bkup.command,
+                         XTRA_BACKUP + PIPE + ZIP + PIPE + ENCRYPT)
+        self.assertEqual(bkup.manifest, "12345.xbstream.gz.enc")
+
+    def test_backup_decrypted_mysqldump_command(self):
+        backupBase.BackupRunner.is_zipped = True
+        backupBase.BackupRunner.is_encrypted = False
+        RunnerClass = utils.import_class(BACKUP_SQLDUMP_CLS)
+        bkup = RunnerClass(12345, user="user", password="password")
+        self.assertEqual(bkup.command, SQLDUMP_BACKUP + PIPE + ZIP)
+        self.assertEqual(bkup.manifest, "12345.gz")
+
+    def test_backup_encrypted_mysqldump_command(self):
+        backupBase.BackupRunner.is_zipped = True
+        backupBase.BackupRunner.is_encrypted = True
+        backupBase.BackupRunner.encrypt_key = CRYPTO_KEY
+        RunnerClass = utils.import_class(BACKUP_SQLDUMP_CLS)
+        bkup = RunnerClass(12345, user="user", password="password")
+        self.assertEqual(bkup.command,
+                         SQLDUMP_BACKUP + PIPE + ZIP + PIPE + ENCRYPT)
+        self.assertEqual(bkup.manifest, "12345.gz.enc")
+
+    def test_restore_decrypted_xtrabackup_command(self):
+        restoreBase.RestoreRunner.is_zipped = True
+        restoreBase.RestoreRunner.is_encrypted = False
+        RunnerClass = utils.import_class(RESTORE_XTRA_CLS)
+        restr = RunnerClass(None, restore_location="/var/lib/mysql")
+        self.assertEqual(restr.restore_cmd, UNZIP + PIPE + XTRA_RESTORE)
+        self.assertEqual(restr.prepare_cmd, PREPARE)
+
+    def test_restore_encrypted_xtrabackup_command(self):
+        restoreBase.RestoreRunner.is_zipped = True
+        restoreBase.RestoreRunner.is_encrypted = True
+        restoreBase.RestoreRunner.decrypt_key = CRYPTO_KEY
+        RunnerClass = utils.import_class(RESTORE_XTRA_CLS)
+        restr = RunnerClass(None, restore_location="/var/lib/mysql")
+        self.assertEqual(restr.restore_cmd,
+                         DECRYPT + PIPE + UNZIP + PIPE + XTRA_RESTORE)
+        self.assertEqual(restr.prepare_cmd, PREPARE)
+
+    def test_restore_decrypted_mysqldump_command(self):
+        restoreBase.RestoreRunner.is_zipped = True
+        restoreBase.RestoreRunner.is_encrypted = False
+        RunnerClass = utils.import_class(RESTORE_SQLDUMP_CLS)
+        restr = RunnerClass(None, restore_location="/var/lib/mysql",
+                            user="user", password="password")
+        self.assertEqual(restr.restore_cmd, UNZIP + PIPE + SQLDUMP_RESTORE)
+        self.assertIsNone(restr.prepare_cmd)
+
+    def test_restore_encrypted_mysqldump_command(self):
+        restoreBase.RestoreRunner.is_zipped = True
+        restoreBase.RestoreRunner.is_encrypted = True
+        restoreBase.RestoreRunner.decrypt_key = CRYPTO_KEY
+        RunnerClass = utils.import_class(RESTORE_SQLDUMP_CLS)
+        restr = RunnerClass(None, restore_location="/var/lib/mysql",
+                            user="user", password="password")
+        self.assertEqual(restr.restore_cmd,
+                         DECRYPT + PIPE + UNZIP + PIPE + SQLDUMP_RESTORE)
+        self.assertIsNone(restr.prepare_cmd)