openstack
/
trove
Code Issues Proposed changes

Addition of DB2 backup & restore functionality 

Implementation of backup and restore functionality for db2
databases. Backup occurs on instance and then it is
compressed and streamed to Swift. Restore works backwards.

Change-Id: I78dd67369a1670ca72a89cc111cae40ed091fe47
Implements: blueprint db2-backup-restore
This commit is contained in:
Ishita Mandhan 2015-11-17 17:37:10 -08:00 committed by mariam john
parent 84d1f54535
commit 421689e14f
9 changed files with 357 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
etc/trove/trove-guestagent.conf.sample
View File

@ -155,3 +155,9 @@ restore_namespace = trove.guestagent.strategies.restore.experimental.couchbase_i
[cassandra]
backup_namespace = trove.guestagent.strategies.backup.experimental.cassandra_impl
restore_namespace = trove.guestagent.strategies.restore.experimental.cassandra_impl
[db2]
# For db2, the following are the defaults for backup, and restore:
# backup_strategy = DB2Backup
# backup_namespace = trove.guestagent.strategies.backup.experimental.db2_impl
# restore_namespace = trove.guestagent.strategies.restore.experimental.db2_impl

18
trove/common/cfg.py
View File

@ -1176,7 +1176,7 @@ db2_opts = [
help='Whether to provision a Cinder volume for datadir.'),
cfg.StrOpt('device_path', default='/dev/vdb',
help='Device path for volume if volume support is enabled.'),
cfg.StrOpt('backup_strategy', default=None,
cfg.StrOpt('backup_strategy', default='DB2Backup',
help='Default strategy to perform backups.'),
cfg.StrOpt('replication_strategy', default=None,
help='Default strategy for replication.'),
@ -1185,10 +1185,18 @@ db2_opts = [
'service during instance-create. The generated password for '
'the root user is immediately returned in the response of '
"instance-create as the 'password' field."),
cfg.StrOpt('backup_namespace', default=None,
help='Namespace to load backup strategies from.'),
cfg.StrOpt('restore_namespace', default=None,
help='Namespace to load restore strategies from.'),
cfg.StrOpt('backup_namespace',
default='trove.guestagent.strategies.backup.experimental.'
'db2_impl',
help='Namespace to load backup strategies from.',
deprecated_name='backup_namespace',
deprecated_group='DEFAULT'),
cfg.StrOpt('restore_namespace',
default='trove.guestagent.strategies.restore.experimental.'
'db2_impl',
help='Namespace to load restore strategies from.',
deprecated_name='restore_namespace',
deprecated_group='DEFAULT'),
cfg.DictOpt('backup_incremental_strategy', default={},
help='Incremental Backup Runner based on the default '
'strategy. For strategies that do not implement an '

20
trove/guestagent/datastore/experimental/db2/manager.py
View File

@ -15,7 +15,10 @@
from oslo_log import log as logging
from trove.common.i18n import _
from trove.common import instance as ds_instance
from trove.common.notification import EndNotification
from trove.guestagent import backup
from trove.guestagent.datastore.experimental.db2 import service
from trove.guestagent.datastore import manager
from trove.guestagent import volume
@ -53,6 +56,8 @@ class Manager(manager.Manager):
self.app.update_hostname()
self.app.change_ownership(mount_point)
self.app.start_db()
if backup_info:
self._perform_restore(backup_info, context, mount_point)
def restart(self, context):
"""
@ -113,3 +118,18 @@ class Manager(manager.Manager):
def start_db_with_conf_changes(self, context, config_contents):
LOG.debug("Starting DB2 with configuration changes.")
self.app.start_db_with_conf_changes(config_contents)
def _perform_restore(self, backup_info, context, restore_location):
LOG.info(_("Restoring database from backup %s.") % backup_info['id'])
try:
backup.restore(context, backup_info, restore_location)
except Exception:
LOG.exception(_("Error performing restore from backup %s.") %
backup_info['id'])
self.status.set_status(ds_instance.ServiceStatuses.FAILED)
raise
LOG.info(_("Restored database successfully."))
def create_backup(self, context, backup_info):
LOG.debug("Creating backup.")
backup.backup(context, backup_info)

11
trove/guestagent/datastore/experimental/db2/system.py
View File

@ -23,6 +23,9 @@ ENABLE_AUTOSTART = (
DISABLE_AUTOSTART = (
"/opt/ibm/db2/V10.5/instance/db2iauto -off " + DB2_INSTANCE_OWNER)
START_DB2 = "db2start"
QUIESCE_DB2 = ("db2 QUIESCE INSTANCE DB2INST1 RESTRICTED ACCESS IMMEDIATE "
"FORCE CONNECTIONS")
UNQUIESCE_DB2 = "db2 UNQUIESCE INSTANCE DB2INST1"
STOP_DB2 = "db2 force application all; db2 terminate; db2stop"
DB2_STATUS = ("ps -ef | grep " + DB2_INSTANCE_OWNER + " | grep db2sysc |"
"grep -v grep | wc -l")
@ -47,3 +50,11 @@ LIST_DB_USERS = (
"db2 +o connect to %(dbname)s; "
"db2 -x select grantee, dataaccessauth from sysibm.sysdbauth; "
"db2 connect reset")
BACKUP_DB = "db2 backup database %(dbname)s to %(dir)s"
RESTORE_DB = (
"db2 restore database %(dbname)s from %(dir)s")
GET_DB_SIZE = (
"db2 connect to %(dbname)s;"
"db2 call get_dbsize_info(?, ?, ?, -1) ")
GET_DB_NAMES = ("find /home/db2inst1/db2inst1/backup/ -type f -name '*.001' |"
" grep -Po \"(?<=backup/)[^.']*(?=\.)\"")

106
trove/guestagent/strategies/backup/experimental/db2_impl.py Normal file
View File

@ -0,0 +1,106 @@
# Copyright 2016 IBM Corp
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log as logging
from trove.common import cfg
from trove.common import exception
from trove.common.i18n import _
from trove.guestagent.common import operating_system
from trove.guestagent.datastore.experimental.db2 import service
from trove.guestagent.datastore.experimental.db2 import system
from trove.guestagent.db import models
from trove.guestagent.strategies.backup import base
CONF = cfg.CONF
DB2_DBPATH = CONF.db2.mount_point
DB2_BACKUP_DIR = DB2_DBPATH + "/backup"
LOG = logging.getLogger(__name__)
class DB2Backup(base.BackupRunner):
"""Implementation of Backup Strategy for DB2."""
__Strategy_name__ = 'db2backup'
def __init__(self, *args, **kwargs):
self.admin = service.DB2Admin()
super(DB2Backup, self).__init__(*args, **kwargs)
def _run_pre_backup(self):
"""Create archival contents in dump dir"""
try:
est_dump_size = self.estimate_dump_size()
avail = operating_system.get_bytes_free_on_fs(DB2_DBPATH)
if est_dump_size > avail:
self.cleanup()
raise OSError(_("Need more free space to backup db2 database,"
" estimated %(est_dump_size)s"
" and found %(avail)s bytes free ") %
{'est_dump_size': est_dump_size,
'avail': avail})
operating_system.create_directory(DB2_BACKUP_DIR,
system.DB2_INSTANCE_OWNER,
system.DB2_INSTANCE_OWNER,
as_root=True)
service.run_command(system.QUIESCE_DB2)
dbNames = self.list_dbnames()
for dbName in dbNames:
service.run_command(system.BACKUP_DB % {
'dbname': dbName, 'dir': DB2_BACKUP_DIR})
service.run_command(system.UNQUIESCE_DB2)
except exception.ProcessExecutionError as e:
LOG.debug("Caught exception when preparing the directory")
self.cleanup()
raise e
@property
def cmd(self):
cmd = 'sudo tar cPf - ' + DB2_BACKUP_DIR
return cmd + self.zip_cmd + self.encrypt_cmd
def cleanup(self):
operating_system.remove(DB2_BACKUP_DIR, force=True, as_root=True)
def _run_post_backup(self):
self.cleanup()
def list_dbnames(self):
dbNames = []
databases, marker = self.admin.list_databases()
for database in databases:
mydb = models.MySQLDatabase()
mydb.deserialize(database)
dbNames.append(mydb.name)
return dbNames
def estimate_dump_size(self):
"""
Estimating the size of the backup based on the size of the data
returned from the get_db_size procedure. The size of the
backup is always going to be smaller than the size of the data.
"""
try:
dbs = self.list_dbnames()
size = 0
for dbname in dbs:
out = service.run_command(system.GET_DB_SIZE % {'dbname':
dbname})
size = size + out
except exception.ProcessExecutionError:
LOG.debug("Error while trying to get db size info")
LOG.debug("Estimated size for databases: " + str(size))
return size

56
trove/guestagent/strategies/restore/experimental/db2_impl.py Normal file
View File

@ -0,0 +1,56 @@
# Copyright 2016 IBM Corp
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log as logging
from trove.common import cfg
from trove.common.i18n import _
from trove.common import utils
from trove.guestagent.common import operating_system
from trove.guestagent.datastore.experimental.db2 import service
from trove.guestagent.datastore.experimental.db2 import system
from trove.guestagent.strategies.restore import base
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
DB2_DBPATH = CONF.db2.mount_point
DB2_BACKUP_DIR = DB2_DBPATH + "/backup"
class DB2Backup(base.RestoreRunner):
"""Implementation of Restore Strategy for DB2."""
__strategy_name__ = 'db2backup'
base_restore_cmd = 'sudo tar xPf -'
def __init__(self, *args, **kwargs):
super(DB2Backup, self).__init__(*args, **kwargs)
self.appStatus = service.DB2AppStatus()
self.app = service.DB2App(self.appStatus)
self.admin = service.DB2Admin()
self.restore_location = DB2_BACKUP_DIR
def post_restore(self):
"""
Restore from the directory that we untarred into
"""
out, err = utils.execute_with_timeout(system.GET_DB_NAMES,
shell=True)
dbNames = out.split()
for dbName in dbNames:
service.run_command(system.RESTORE_DB % {'dbname': dbName,
'dir': DB2_BACKUP_DIR})
LOG.info(_("Cleaning out restore location post: %s."), DB2_BACKUP_DIR)
operating_system.remove(DB2_BACKUP_DIR, force=True, as_root=True)

12
trove/tests/unittests/backup/test_backupagent.py
View File

@ -31,6 +31,7 @@ from trove.guestagent.common.configuration import ImportOverrideStrategy
from trove.guestagent.strategies.backup.base import BackupRunner
from trove.guestagent.strategies.backup.base import UnknownBackupType
from trove.guestagent.strategies.backup.experimental import couchbase_impl
from trove.guestagent.strategies.backup.experimental import db2_impl
from trove.guestagent.strategies.backup.experimental import mongo_impl
from trove.guestagent.strategies.backup.experimental import redis_impl
from trove.guestagent.strategies.backup import mysql_impl
@ -251,6 +252,17 @@ class BackupAgentTest(trove_testtools.TestCase):
self.assertIsNotNone(cbbackup.manifest)
self.assertIn('gz.enc', cbbackup.manifest)
def test_backup_impl_DB2Backup(self):
netutils.get_my_ipv4 = Mock(return_value="1.1.1.1")
db2_backup = db2_impl.DB2Backup('db2backup', extra_opts='')
self.assertIsNotNone(db2_backup)
str_db2_backup_cmd = ("sudo tar cPf - /home/db2inst1/db2inst1/backup "
"| gzip | openssl enc -aes-256-cbc -salt -pass "
"pass:default_aes_cbc_key")
self.assertEqual(str_db2_backup_cmd, db2_backup.cmd)
self.assertIsNotNone(db2_backup.manifest)
self.assertIn('gz.enc', db2_backup.manifest)
@mock.patch.object(ImportOverrideStrategy, '_initialize_import_directory')
def test_backup_impl_MongoDump(self, _):
netutils.get_my_ipv4 = Mock(return_value="1.1.1.1")

114
trove/tests/unittests/guestagent/test_backups.py
View File

@ -56,6 +56,11 @@ BACKUP_NODETOOLSNAPSHOT_CLS = ("trove.guestagent.strategies.backup."
"experimental.cassandra_impl.NodetoolSnapshot")
RESTORE_NODETOOLSNAPSHOT_CLS = ("trove.guestagent.strategies.restore."
"experimental.cassandra_impl.NodetoolSnapshot")
BACKUP_DB2_CLS = ("trove.guestagent.strategies.backup."
"experimental.db2_impl.DB2Backup")
RESTORE_DB2_CLS = ("trove.guestagent.strategies.restore."
"experimental.db2_impl.DB2Backup")
PIPE = " | "
ZIP = "gzip"
@ -98,6 +103,9 @@ MONGODUMP_RESTORE = "sudo tar xPf -"
REDISBACKUP_CMD = "sudo cat /var/lib/redis/dump.rdb"
REDISBACKUP_RESTORE = "tee /var/lib/redis/dump.rdb"
DB2BACKUP_CMD = "sudo tar cPf - /home/db2inst1/db2inst1/backup"
DB2BACKUP_RESTORE = "sudo tar xPf -"
class GuestAgentBackupTest(trove_testtools.TestCase):
@ -418,6 +426,45 @@ class GuestAgentBackupTest(trove_testtools.TestCase):
self.assertEqual(restr.restore_cmd,
DECRYPT + PIPE + UNZIP + PIPE + REDISBACKUP_RESTORE)
@patch.object(utils, 'execute_with_timeout')
def test_backup_encrypted_db2backup_command(self, *mock):
backupBase.BackupRunner.is_encrypted = True
backupBase.BackupRunner.encrypt_key = CRYPTO_KEY
RunnerClass = utils.import_class(BACKUP_DB2_CLS)
bkp = RunnerClass(12345) # this is not db2 backup filename
self.assertIsNotNone(12345) # look into this
self.assertEqual(
DB2BACKUP_CMD + PIPE + ZIP + PIPE + ENCRYPT, bkp.command)
self.assertIn("gz.enc", bkp.manifest)
@patch.object(utils, 'execute_with_timeout')
def test_backup_not_encrypted_db2backup_command(self, *mock):
backupBase.BackupRunner.is_encrypted = False
backupBase.BackupRunner.encrypt_key = CRYPTO_KEY
RunnerClass = utils.import_class(BACKUP_DB2_CLS)
bkp = RunnerClass(12345)
self.assertIsNotNone(bkp)
self.assertEqual(DB2BACKUP_CMD + PIPE + ZIP, bkp.command)
self.assertIn("gz", bkp.manifest)
def test_restore_decrypted_db2backup_command(self):
restoreBase.RestoreRunner.is_zipped = True
restoreBase.RestoreRunner.is_encrypted = False
RunnerClass = utils.import_class(RESTORE_DB2_CLS)
restr = RunnerClass(None, restore_location="/tmp",
location="filename", checksum="md5")
self.assertEqual(restr.restore_cmd, UNZIP + PIPE + DB2BACKUP_RESTORE)
def test_restore_encrypted_db2backup_command(self):
restoreBase.RestoreRunner.is_zipped = True
restoreBase.RestoreRunner.is_encrypted = True
restoreBase.RestoreRunner.encrypt_key = CRYPTO_KEY
RunnerClass = utils.import_class(RESTORE_DB2_CLS)
restr = RunnerClass(None, restore_location="/tmp",
location="filename", checksum="md5")
self.assertEqual(restr.restore_cmd,
DECRYPT + PIPE + UNZIP + PIPE + DB2BACKUP_RESTORE)
class CassandraBackupTest(trove_testtools.TestCase):
@ -796,3 +843,70 @@ class RedisRestoreTests(trove_testtools.TestCase):
exception.ProcessExecutionError('Error'))
self.assertRaises(exception.ProcessExecutionError,
self.restore_runner.restore)
class DB2BackupTests(trove_testtools.TestCase):
def setUp(self):
super(DB2BackupTests, self).setUp()
self.exec_timeout_patch = patch.object(utils, 'execute_with_timeout')
self.exec_timeout_patch.start()
self.backup_runner = utils.import_class(BACKUP_DB2_CLS)
self.backup_runner_patch = patch.multiple(
self.backup_runner, _run=DEFAULT,
_run_pre_backup=DEFAULT, _run_post_backup=DEFAULT)
def tearDown(self):
super(DB2BackupTests, self).tearDown()
self.backup_runner_patch.stop()
self.exec_timeout_patch.stop()
def test_backup_success(self):
backup_runner_mocks = self.backup_runner_patch.start()
with self.backup_runner(12345):
pass
backup_runner_mocks['_run_pre_backup'].assert_called_once_with()
backup_runner_mocks['_run'].assert_called_once_with()
backup_runner_mocks['_run_post_backup'].assert_called_once_with()
def test_backup_failed_due_to_run_backup(self):
backup_runner_mocks = self.backup_runner_patch.start()
backup_runner_mocks['_run'].configure_mock(
side_effect=exception.TroveError('test'))
with ExpectedException(exception.TroveError, 'test'):
with self.backup_runner(12345):
pass
backup_runner_mocks['_run_pre_backup'].assert_called_once_with()
backup_runner_mocks['_run'].assert_called_once_with()
self.assertEqual(0, backup_runner_mocks['_run_post_backup'].call_count)
class DB2RestoreTests(trove_testtools.TestCase):
def setUp(self):
super(DB2RestoreTests, self).setUp()
self.restore_runner = utils.import_class(
RESTORE_DB2_CLS)('swift', location='http://some.where',
checksum='True_checksum',
restore_location='/var/lib/somewhere')
def tearDown(self):
super(DB2RestoreTests, self).tearDown()
def test_restore_success(self):
expected_content_length = 123
self.restore_runner._run_restore = mock.Mock(
return_value=expected_content_length)
self.restore_runner.post_restore = mock.Mock()
actual_content_length = self.restore_runner.restore()
self.assertEqual(
expected_content_length, actual_content_length)
def test_restore_failed_due_to_run_restore(self):
self.restore_runner._run_restore = mock.Mock(
side_effect=exception.ProcessExecutionError('Error'))
self.restore_runner.post_restore = mock.Mock()
self.assertRaises(exception.ProcessExecutionError,
self.restore_runner.restore)

20
trove/tests/unittests/guestagent/test_db2_manager.py
View File

@ -17,6 +17,7 @@ from mock import patch
from testtools.matchers import Is, Equals, Not
from trove.common.instance import ServiceStatuses
from trove.guestagent import backup
from trove.guestagent.datastore.experimental.db2 import (
manager as db2_manager)
from trove.guestagent.datastore.experimental.db2 import (
@ -56,6 +57,7 @@ class GuestAgentDB2ManagerTest(trove_testtools.TestCase):
self.orig_list_users = db2_service.DB2Admin.list_users
self.orig_delete_user = db2_service.DB2Admin.delete_user
self.orig_update_hostname = db2_service.DB2App.update_hostname
self.orig_backup_restore = backup.restore
def tearDown(self):
super(GuestAgentDB2ManagerTest, self).tearDown()
@ -75,6 +77,7 @@ class GuestAgentDB2ManagerTest(trove_testtools.TestCase):
db2_service.DB2Admin.list_users = self.orig_list_users
db2_service.DB2Admin.delete_user = self.orig_delete_user
db2_service.DB2App.update_hostname = self.orig_update_hostname
backup.restore = self.orig_backup_restore
def test_update_status(self):
mock_status = MagicMock()
@ -91,9 +94,18 @@ class GuestAgentDB2ManagerTest(trove_testtools.TestCase):
def test_prepare_database(self):
self._prepare_dynamic(databases=['db1'])
def test_prepare_from_backup(self):
self._prepare_dynamic(['db2'], backup_id='123backup')
def _prepare_dynamic(self, packages=None, databases=None, users=None,
config_content=None, device_path='/dev/vdb',
is_db_installed=True, backup_id=None, overrides=None):
backup_info = {'id': backup_id,
'location': 'fake-location',
'type': 'DB2Backup',
'checksum': 'fake-checksum'} if backup_id else None
mock_status = MagicMock()
mock_app = MagicMock()
self.manager.appStatus = mock_status
@ -109,6 +121,7 @@ class GuestAgentDB2ManagerTest(trove_testtools.TestCase):
volume.VolumeDevice.mount_points = MagicMock(return_value=[])
db2_service.DB2Admin.create_user = MagicMock(return_value=None)
db2_service.DB2Admin.create_database = MagicMock(return_value=None)
backup.restore = MagicMock(return_value=None)
with patch.object(pkg.Package, 'pkg_is_installed',
return_value=MagicMock(
@ -119,7 +132,7 @@ class GuestAgentDB2ManagerTest(trove_testtools.TestCase):
memory_mb='2048', users=users,
device_path=device_path,
mount_point="/home/db2inst1/db2inst1",
backup_info=None,
backup_info=backup_info,
overrides=None,
cluster_config=None)
@ -135,6 +148,11 @@ class GuestAgentDB2ManagerTest(trove_testtools.TestCase):
else:
self.assertFalse(db2_service.DB2Admin.create_user.called)
if backup_id:
backup.restore.assert_any_call(self.context,
backup_info,
'/home/db2inst1/db2inst1')
def test_restart(self):
mock_status = MagicMock()
self.manager.appStatus = mock_status