openstack/trove
Code Issues Proposed changes

Support incremental backup for MariaDB

Browse Source 
Enable the tests in CI as well.

Change-Id: Ie9706d26355bd325baf50ec874f05e6904768a1a
This commit is contained in:
Lingxian Kong 2019-12-07 22:49:08 +13:00
parent e226ba68ab
commit 605ff34608
8 changed files with 154 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
devstack/plugin.sh
View File

@ -239,7 +239,6 @@ function configure_trove {
iniset $TROVE_CONF DEFAULT remote_nova_client trove.common.clients_admin.nova_client_trove_admin
iniset $TROVE_CONF DEFAULT remote_cinder_client trove.common.clients_admin.cinder_client_trove_admin
iniset $TROVE_CONF DEFAULT remote_neutron_client trove.common.clients_admin.neutron_client_trove_admin
iniset $TROVE_CONF DEFAULT remote_swift_client trove.common.clients_admin.swift_client_trove_admin
iniset $TROVE_CONF DEFAULT remote_glance_client trove.common.clients_admin.glance_client_trove_admin
iniset $TROVE_CONF cassandra tcp_ports 7000,7001,7199,9042,9160
@ -278,7 +277,6 @@ function configure_trove {
iniset $TROVE_GUESTAGENT_CONF DEFAULT remote_nova_client trove.common.clients_admin.nova_client_trove_admin
iniset $TROVE_GUESTAGENT_CONF DEFAULT remote_cinder_client trove.common.clients_admin.cinder_client_trove_admin
iniset $TROVE_GUESTAGENT_CONF DEFAULT remote_neutron_client trove.common.clients_admin.neutron_client_trove_admin
iniset $TROVE_GUESTAGENT_CONF DEFAULT remote_swift_client trove.common.clients_admin.swift_client_trove_admin
iniset $TROVE_GUESTAGENT_CONF DEFAULT remote_glance_client trove.common.clients_admin.glance_client_trove_admin
# 1. To avoid 'Connection timed out' error of sudo command inside the guest agent

1
trove/guestagent/backup/backupagent.py
View File

@ -166,6 +166,7 @@ class BackupAgent(object):
LOG.info("Restoring instance from backup %(id)s to "
"%(restore_location)s", backup_info)
content_size = runner.restore()
LOG.info("Restore from backup %(id)s completed successfully "
"to %(restore_location)s", backup_info)

46
trove/guestagent/strategies/backup/experimental/mariadb_impl.py
View File

@ -15,6 +15,7 @@ import re
from oslo_log import log as logging
from trove.common.i18n import _
from trove.guestagent.datastore.mysql import service as mysql_service
from trove.guestagent.datastore.mysql_common import service as common_service
from trove.guestagent.strategies.backup import base
@ -29,13 +30,13 @@ class MariaBackup(base.BackupRunner):
@property
def user_and_pass(self):
return (' --user=%(user)s --password=%(password)s --host=127.0.0.1 ' %
return ('--user=%(user)s --password=%(password)s --host=127.0.0.1' %
{'user': common_service.ADMIN_USER_NAME,
'password': mysql_service.MySqlApp.get_auth_password()})
@property
def cmd(self):
cmd = ('sudo mariabackup --backup --stream=xbstream' +
cmd = ('sudo mariabackup --backup --stream=xbstream ' +
self.user_and_pass + ' 2>' + BACKUP_LOG)
return cmd + self.zip_cmd + self.encrypt_cmd
@ -61,10 +62,49 @@ class MariaBackup(base.BackupRunner):
return True
def metadata(self):
LOG.debug('Getting metadata for backup %s', self.base_filename)
meta = {}
lsn = re.compile(r"The latest check point \(for incremental\): "
r"'(\d+)'")
with open(BACKUP_LOG, 'r') as backup_log:
output = backup_log.read()
match = lsn.search(output)
if match:
meta = {'lsn': match.group(1)}
LOG.info("Metadata for backup %s: %s", self.base_filename, meta)
return meta
@property
def filename(self):
return '%s.xbstream' % self.base_filename
class MariaBackupIncremental(MariaBackup):
pass
def __init__(self, *args, **kwargs):
if not kwargs.get('lsn'):
raise AttributeError(_('lsn attribute missing, bad parent?'))
super(MariaBackupIncremental, self).__init__(*args, **kwargs)
self.parent_location = kwargs.get('parent_location')
self.parent_checksum = kwargs.get('parent_checksum')
@property
def cmd(self):
cmd = (
'sudo mariabackup --backup --stream=xbstream'
' --incremental-lsn=%(lsn)s ' +
self.user_and_pass +
' 2>' +
BACKUP_LOG
)
return cmd + self.zip_cmd + self.encrypt_cmd
def metadata(self):
meta = super(MariaBackupIncremental, self).metadata()
meta.update({
'parent_location': self.parent_location,
'parent_checksum': self.parent_checksum,
})
return meta

4
trove/guestagent/strategies/backup/mysql_impl.py
View File

@ -100,7 +100,7 @@ class InnoBackupEx(base.BackupRunner):
return True
def metadata(self):
LOG.debug('Getting metadata from backup.')
LOG.debug('Getting metadata for backup %s', self.base_filename)
meta = {}
lsn = re.compile(r"The latest check point \(for incremental\): "
r"'(\d+)'")
@ -109,7 +109,7 @@ class InnoBackupEx(base.BackupRunner):
match = lsn.search(output)
if match:
meta = {'lsn': match.group(1)}
LOG.info("Metadata for backup: %s.", str(meta))
LOG.info("Metadata for backup %s: %s", self.base_filename, meta)
return meta
@property

105
trove/guestagent/strategies/restore/experimental/mariadb_impl.py
View File

@ -11,22 +11,32 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import glob
import os
from oslo_log import log as logging
from trove.common import cfg
from trove.common import utils
from trove.guestagent.common import operating_system
from trove.guestagent.datastore.experimental.mariadb import service
from trove.guestagent.datastore.mysql_common import service as mysql_service
from trove.guestagent.strategies.restore import base
from trove.guestagent.strategies.restore import mysql_impl
LOG = logging.getLogger(__name__)
PREPARE_LOG = '/tmp/innoprepare.log'
class MariaBackup(mysql_impl.InnoBackupEx):
class MariaBackup(base.RestoreRunner, mysql_impl.MySQLRestoreMixin):
__strategy_name__ = 'mariabackup'
base_restore_cmd = ('sudo mbstream -x -C %(restore_location)s '
'2>/tmp/xbstream_extract.log')
def __init__(self, *args, **kwargs):
self._app = None
super(MariaBackup, self).__init__(*args, **kwargs)
@property
def app(self):
if self._app is None:
@ -35,6 +45,14 @@ class MariaBackup(mysql_impl.InnoBackupEx):
)
return self._app
def pre_restore(self):
self.app.stop_db()
LOG.debug("Cleaning out restore location: %s.", self.restore_location)
operating_system.chmod(self.restore_location,
operating_system.FileMode.SET_FULL,
as_root=True)
utils.clean_out(self.restore_location)
def post_restore(self):
operating_system.chown(self.restore_location, 'mysql', None,
force=True, as_root=True)
@ -45,6 +63,11 @@ class MariaBackup(mysql_impl.InnoBackupEx):
self.app.start_mysql()
LOG.debug("Finished post restore.")
def _delete_old_binlogs(self):
files = glob.glob(os.path.join(self.restore_location, "ib_logfile*"))
for f in files:
os.unlink(f)
def check_process(self):
LOG.debug('Checking return code of mbstream restore process.')
return_code = self.process.wait()
@ -56,4 +79,82 @@ class MariaBackup(mysql_impl.InnoBackupEx):
class MariaBackupIncremental(MariaBackup):
pass
__strategy_name__ = 'mariabackupincremental'
incremental_prep = ('sudo mariabackup --prepare '
'--target-dir=%(restore_location)s '
'%(incremental_args)s '
'2>/tmp/innoprepare.log')
def __init__(self, *args, **kwargs):
super(MariaBackupIncremental, self).__init__(*args, **kwargs)
self.content_length = 0
def _incremental_restore_cmd(self, incremental_dir):
"""Return a command for a restore with a incremental location."""
args = {'restore_location': incremental_dir}
return (self.decrypt_cmd +
self.unzip_cmd +
(self.base_restore_cmd % args))
def _incremental_prepare_cmd(self, incremental_dir):
if incremental_dir is not None:
incremental_arg = '--incremental-dir=%s' % incremental_dir
else:
incremental_arg = ''
args = {
'restore_location': self.restore_location,
'incremental_args': incremental_arg,
}
return self.incremental_prep % args
def _incremental_prepare(self, incremental_dir):
prepare_cmd = self._incremental_prepare_cmd(incremental_dir)
LOG.debug("Running mariabackup prepare: %s.", prepare_cmd)
utils.execute(prepare_cmd, shell=True)
LOG.debug("mariabackup prepare finished successfully.")
def _incremental_restore(self, location, checksum):
"""Recursively apply backups from all parents.
If we are the parent then we restore to the restore_location and
we apply the logs to the restore_location only.
Otherwise if we are an incremental we restore to a subfolder to
prevent stomping on the full restore data. Then we run apply log
with the '--incremental-dir' flag
"""
metadata = self.storage.load_metadata(location, checksum)
incremental_dir = None
if 'parent_location' in metadata:
LOG.info("Restoring parent: %(parent_location)s"
" checksum: %(parent_checksum)s.", metadata)
parent_location = metadata['parent_location']
parent_checksum = metadata['parent_checksum']
# Restore parents recursively so backup are applied sequentially
self._incremental_restore(parent_location, parent_checksum)
# for *this* backup set the incremental_dir
# just use the checksum for the incremental path as it is
# sufficiently unique /var/lib/mysql/<checksum>
incremental_dir = os.path.join(
cfg.get_configuration_property('mount_point'), checksum)
operating_system.create_directory(incremental_dir, as_root=True)
command = self._incremental_restore_cmd(incremental_dir)
else:
# The parent (full backup) use the same command from InnobackupEx
# super class and do not set an incremental_dir.
command = self.restore_cmd
self.content_length += self._unpack(location, checksum, command)
self._incremental_prepare(incremental_dir)
# Delete unpacked incremental backup metadata
if incremental_dir:
operating_system.remove(incremental_dir, force=True, as_root=True)
def _run_restore(self):
"""Run incremental restore."""
self._incremental_restore(self.location, self.checksum)
return self.content_length

5
trove/guestagent/strategies/restore/mysql_impl.py
View File

@ -211,8 +211,7 @@ class InnoBackupEx(base.RestoreRunner, MySQLRestoreMixin):
def pre_restore(self):
self.app.stop_db()
LOG.info("Cleaning out restore location: %s.",
self.restore_location)
LOG.debug("Cleaning out restore location: %s.", self.restore_location)
operating_system.chmod(self.restore_location, FileMode.SET_FULL,
as_root=True)
utils.clean_out(self.restore_location)
@ -313,7 +312,7 @@ class InnoBackupExIncremental(InnoBackupEx):
prepare_cmd = self._incremental_prepare_cmd(incremental_dir)
LOG.debug("Running innobackupex prepare: %s.", prepare_cmd)
utils.execute(prepare_cmd, shell=True)
LOG.info("Innobackupex prepare finished successfully.")
LOG.debug("Innobackupex prepare finished successfully.")
def _incremental_restore(self, location, checksum):
"""Recursively apply backups from all parents.

3
trove/taskmanager/models.py
View File

@ -1089,7 +1089,8 @@ class BuiltInstanceTasks(BuiltInstance, NotifyMixin, ConfigurationMixin):
action.execute()
def create_backup(self, backup_info):
LOG.info("Initiating backup for instance %s.", self.id)
LOG.info("Initiating backup for instance %s, backup_info: %s", self.id,
backup_info)
self.guest.create_backup(backup_info)
def backup_required_for_replication(self):

2
trove/tests/int_tests.py
View File

@ -319,7 +319,7 @@ register(
["mariadb_supported"],
single=[common_groups,
backup_groups,
# backup_incremental_groups,
backup_incremental_groups,
configuration_groups,
database_actions_groups,
root_actions_groups,