openstack/freezer
Code Issues Proposed changes

Refactoring of mysql, msserver and mongo code.

Browse Source 
Implements blueprint: application-hook-refactoring

Change-Id: I6df7ae22ec17479dc926cc4e764091487544b5fb
This commit is contained in:
eldar nugaev 2016-02-16 10:59:29 +00:00
parent 77453bdae7
commit e001119d9e
15 changed files with 291 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
freezer/backup.py
View File

@ -18,7 +18,6 @@ Freezer Backup modes related functions
import os
import time
from freezer import config
from freezer import lvm
from freezer import utils
from freezer import vss
@ -31,93 +30,6 @@ logging = log.getLogger(__name__)
home = os.path.expanduser("~")
def backup_mode_sql_server(backup_opt_dict, storage):
"""
Execute a SQL Server DB backup. Currently only backups with shadow
copy are supported. This mean, as soon as the shadow copy is created
the db writes will be blocked and a checkpoint will be created, as soon
as the backup finish the db will be unlocked and the backup will be
uploaded. A sql_server.conf_file is required for this operation.
"""
with open(backup_opt_dict.sql_server_conf, 'r') as sql_conf_file_fd:
parsed_config = config.ini_parse(sql_conf_file_fd.read())
sql_server_instance = parsed_config["instance"]
# Dirty hack - please remove any modification of backup_opt_dict
backup_opt_dict.sql_server_instance = sql_server_instance
try:
winutils.stop_sql_server(sql_server_instance)
return backup(backup_opt_dict, storage, backup_opt_dict.engine)
finally:
if not backup_opt_dict.snapshot:
# if snapshot is false, wait until the backup is complete
# to start sql server again
winutils.start_sql_server(sql_server_instance)
def backup_mode_mysql(backup_opt_dict, storage):
"""
Execute a MySQL DB backup. currently only backup with lvm snapshots
are supported. This mean, just before the lvm snap vol is created,
the db tables will be flushed and locked for read, then the lvm create
command will be executed and after that, the table will be unlocked and
the backup will be executed. It is important to have the available in
backup_args.mysql_conf the file where the database host, name, user,
password and port are set.
"""
try:
import pymysql as MySQLdb
except ImportError:
raise ImportError('Please install PyMySQL module')
if not backup_opt_dict.mysql_conf:
raise ValueError('MySQL: please provide a valid config file')
with open(backup_opt_dict.mysql_conf, 'r') as mysql_file_fd:
parsed_config = config.ini_parse(mysql_file_fd.read())
# Initialize the DB object and connect to the db according to
# the db mysql backup file config
try:
backup_opt_dict.mysql_db_inst = MySQLdb.connect(
host=parsed_config.get("host", False),
port=int(parsed_config.get("port", 3306)),
user=parsed_config.get("user", False),
passwd=parsed_config.get("password", False))
except Exception as error:
raise Exception('[*] MySQL: {0}'.format(error))
# Execute backup
return backup(backup_opt_dict, storage, backup_opt_dict.engine)
def backup_mode_mongo(backup_opt_dict, storage):
"""
Execute the necessary tasks for file system backup mode
"""
try:
import pymongo
except ImportError:
raise ImportError('please install pymongo module')
logging.info('[*] MongoDB backup is being executed...')
logging.info('[*] Checking is the localhost is Master/Primary...')
mongodb_port = '27017'
local_hostname = backup_opt_dict.hostname
db_host_port = '{0}:{1}'.format(local_hostname, mongodb_port)
mongo_client = pymongo.MongoClient(db_host_port)
master_dict = dict(mongo_client.admin.command("isMaster"))
mongo_me = master_dict['me']
mongo_primary = master_dict['primary']
if mongo_me == mongo_primary:
return backup(backup_opt_dict, storage, backup_opt_dict.engine)
else:
logging.warning('[*] localhost {0} is not Master/Primary,\
exiting...'.format(local_hostname))
return None
class BackupOs:
def __init__(self, client_manager, container, storage):
@ -225,16 +137,9 @@ def snapshot_create(backup_opt_dict):
backup_opt_dict.path_to_backup = winutils.use_shadow(
backup_opt_dict.path_to_backup,
backup_opt_dict.windows_volume)
# execute this after the snapshot creation
if backup_opt_dict.mode == 'sqlserver':
winutils.start_sql_server(backup_opt_dict.sql_server_instance)
return True
return False
else:
return lvm.lvm_snap(backup_opt_dict)
@ -247,7 +152,7 @@ def snapshot_remove(backup_opt_dict, shadow, windows_volume):
lvm.lvm_snap_remove(backup_opt_dict)
def backup(backup_opt_dict, storage, engine):
def backup(backup_opt_dict, storage, engine, app_mode):
"""
:param backup_opt_dict:
@ -255,6 +160,7 @@ def backup(backup_opt_dict, storage, engine):
:type storage: freezer.storage.base.Storage
:param engine: Backup Engine
:type engine: freezer.engine.engine.BackupEngine
:type app_mode: freezer.mode.mode.Mode
:return:
"""
backup_media = backup_opt_dict.backup_media
@ -263,8 +169,10 @@ def backup(backup_opt_dict, storage, engine):
backup_opt_dict.time_stamp = time_stamp
if backup_media == 'fs':
app_mode.prepare()
snapshot_taken = snapshot_create(backup_opt_dict)
if snapshot_taken:
app_mode.release()
try:
filepath = '.'
chdir_path = os.path.expanduser(
@ -285,6 +193,7 @@ def backup(backup_opt_dict, storage, engine):
return backup_instance
finally:
# whether an error occurred or not, remove the snapshot anyway
app_mode.release()
if snapshot_taken:
snapshot_remove(backup_opt_dict, backup_opt_dict.shadow,
backup_opt_dict.windows_volume)

2
freezer/common/config.py
View File

@ -59,7 +59,7 @@ DEFAULT_PARAMS = {
'upload_limit': -1, 'always_level': False, 'version': False,
'dry_run': False, 'lvm_snapsize': DEFAULT_LVM_SNAPSIZE,
'restore_abs_path': False, 'log_file': None, 'log_level': "info",
'mode': 'fs', 'action': 'backup', 'shadow': '', 'shadow_path': '',
'mode': 'default', 'action': 'backup', 'shadow': '', 'shadow_path': '',
'windows_volume': '', 'command': None, 'metadata_out': False,
'storage': 'swift', 'ssh_key': '', 'ssh_username': '', 'ssh_host': '',
'ssh_port': DEFAULT_SSH_PORT, 'compression': 'gzip'

25
freezer/job.py
View File

@ -16,6 +16,7 @@ limitations under the License.
"""
import datetime
from oslo_utils import importutils
import sys
import time
@ -78,21 +79,13 @@ class BackupJob(Job):
logging.error('Error while sync exec: {0}'.format(err))
except Exception as error:
logging.error('Error while sync exec: {0}'.format(error))
if self.conf.mode == 'fs':
backup_instance = \
backup.backup(self.conf, self.storage, self.engine)
elif self.conf.mode == 'mongo':
backup_instance = \
backup.backup_mode_mongo(self.conf, self.storage)
elif self.conf.mode == 'mysql':
backup_instance = \
backup.backup_mode_mysql(self.conf, self.storage)
elif self.conf.mode == 'sqlserver':
backup_instance = \
backup.backup_mode_sql_server(self.conf, self.storage)
else:
raise ValueError('Please provide a valid backup mode')
if not self.conf.mode:
raise ValueError("Empty mode")
mod_name = 'freezer.mode.{0}.{1}'.format(
self.conf.mode, self.conf.mode.capitalize() + 'Mode')
app_mode = importutils.import_object(mod_name, self.conf)
backup_instance = backup.backup(
self.conf, self.storage, self.engine, app_mode)
level = backup_instance.level if backup_instance else 0
@ -133,7 +126,7 @@ class RestoreJob(Job):
restore_timestamp)
self.engine.restore(backup, restore_abs_path)
return
return {}
res = restore.RestoreOs(conf.client_manager, conf.container)
if conf.backup_media == 'nova':

15
freezer/lvm.py
View File

@ -126,27 +126,12 @@ def lvm_snap(backup_opt_dict):
backup_opt_dict.lvm_snapname,
backup_opt_dict.lvm_srcvol))
# If backup mode is mysql, then the db will be flushed and read locked
# before the creation of the lvm snap
if backup_opt_dict.mode == 'mysql':
cursor = backup_opt_dict.mysql_db_inst.cursor()
cursor.execute('FLUSH TABLES WITH READ LOCK')
backup_opt_dict.mysql_db_inst.commit()
lvm_process = subprocess.Popen(
lvm_create_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True,
executable=utils.find_executable('bash'))
(lvm_out, lvm_err) = lvm_process.communicate()
# Unlock MySQL Tables if backup is == mysql
# regardless of the snapshot being taken or not
if backup_opt_dict.mode == 'mysql':
cursor.execute('UNLOCK TABLES')
backup_opt_dict.mysql_db_inst.commit()
cursor.close()
backup_opt_dict.mysql_db_inst.close()
if lvm_process.returncode:
raise Exception('lvm snapshot creation error: {0}'.format(lvm_err))

0
freezer/mode/__init__.py Normal file
View File

35
freezer/mode/default.py Normal file
View File

@ -0,0 +1,35 @@
# (c) Copyright 2015,2016 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from freezer.mode import mode
class DefaultMode(mode.Mode):
def __init__(self, conf):
pass
@property
def name(self):
return "default"
@property
def version(self):
return "1.0"
def release(self):
pass
def prepare(self):
pass

35
freezer/mode/mode.py Normal file
View File

@ -0,0 +1,35 @@
# (c) Copyright 2015,2016 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import abc
import six
@six.add_metaclass(abc.ABCMeta)
class Mode(object):
@abc.abstractproperty
def name(self):
pass
@abc.abstractproperty
def version(self):
pass
@abc.abstractmethod
def prepare(self):
pass
@abc.abstractmethod
def release(self):
pass

55
freezer/mode/mongo.py Normal file
View File

@ -0,0 +1,55 @@
# (c) Copyright 2015,2016 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from freezer.mode import mode
class MongoDbMode(mode.Mode):
"""
Execute the necessary tasks for file system backup mode
"""
@property
def name(self):
return "mongo"
@property
def version(self):
return "1.0"
def release(self):
pass
def prepare(self):
pass
def __init__(self, conf):
try:
import pymongo
except ImportError:
raise ImportError('please install pymongo module')
logging.info('[*] MongoDB backup is being executed...')
logging.info('[*] Checking is the localhost is Master/Primary...')
# todo unhardcode this
mongodb_port = '27017'
local_hostname = conf.hostname
db_host_port = '{0}:{1}'.format(local_hostname, mongodb_port)
mongo_client = pymongo.MongoClient(db_host_port)
master_dict = dict(mongo_client.admin.command("isMaster"))
if master_dict['me'] != master_dict['primary']:
raise Exception('[*] localhost {0} is not Master/Primary,\
exiting...'.format(local_hostname))

71
freezer/mode/mysql.py Normal file
View File

@ -0,0 +1,71 @@
# (c) Copyright 2015,2016 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from freezer import config
from freezer.mode import mode
class MysqlMode(mode.Mode):
"""
Execute a MySQL DB backup. currently only backup with lvm snapshots
are supported. This mean, just before the lvm snap vol is created,
the db tables will be flushed and locked for read, then the lvm create
command will be executed and after that, the table will be unlocked and
the backup will be executed. It is important to have the available in
backup_args.mysql_conf the file where the database host, name, user,
password and port are set.
"""
@property
def name(self):
return "mysql"
@property
def version(self):
return "1.0"
def release(self):
if not self.released:
self.released = True
self.cursor.execute('UNLOCK TABLES')
self.mysql_db_inst.commit()
self.cursor.close()
self.mysql_db_inst.close()
def prepare(self):
self.released = False
self.cursor = self.mysql_db_inst.cursor()
self.cursor.execute('FLUSH TABLES WITH READ LOCK')
self.mysql_db_inst.commit()
def __init__(self, conf):
try:
import pymysql as MySQLdb
except ImportError:
raise ImportError('Please install PyMySQL module')
with open(conf.mysql_conf, 'r') as mysql_file_fd:
parsed_config = config.ini_parse(mysql_file_fd.read())
# Initialize the DB object and connect to the db according to
# the db mysql backup file config
self.released = False
try:
self.mysql_db_inst = MySQLdb.connect(
host=parsed_config.get("host", False),
port=int(parsed_config.get("port", 3306)),
user=parsed_config.get("user", False),
passwd=parsed_config.get("password", False))
self.cursor = None
except Exception as error:
raise Exception('[*] MySQL: {0}'.format(error))

77
freezer/mode/sqlserver.py Normal file
View File

@ -0,0 +1,77 @@
# (c) Copyright 2015,2016 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from freezer import config
from freezer.mode import mode
from freezer import utils
from freezer import winutils
class SqlserverMode(mode.Mode):
"""
Execute a SQL Server DB backup. Currently only backups with shadow
copy are supported. This mean, as soon as the shadow copy is created
the db writes will be blocked and a checkpoint will be created, as soon
as the backup finish the db will be unlocked and the backup will be
uploaded. A sql_server.conf_file is required for this operation.
"""
def __init__(self, conf):
self.released = False
with open(conf.sql_server_conf, 'r') as sql_conf_file_fd:
self.sql_server_instance = \
config.ini_parse(sql_conf_file_fd.read())["instance"]
@property
def name(self):
return "sqlserver"
@property
def version(self):
return "1.0"
def stop_sql_server(self):
""" Stop a SQL Server instance to
perform the backup of the db files """
logging.info('[*] Stopping SQL Server for backup')
with winutils.DisableFileSystemRedirection():
cmd = 'net stop "SQL Server ({0})"'\
.format(self.sql_server_instance)
(out, err) = utils.create_subprocess(cmd)
if err != '':
raise Exception('[*] Error while stopping SQL Server,'
', error {0}'.format(err))
def start_sql_server(self):
""" Start the SQL Server instance after the backup is completed """
with winutils.DisableFileSystemRedirection():
cmd = 'net start "SQL Server ({0})"'.format(
self.sql_server_instance)
(out, err) = utils.create_subprocess(cmd)
if err != '':
raise Exception('[*] Error while starting SQL Server'
', error {0}'.format(err))
logging.info('[*] SQL Server back to normal')
def prepare(self):
self.stop_sql_server()
self.released = False
def release(self):
if not self.released:
self.released = True
self.start_sql_server()

4
freezer/tests/commons.py
View File

@ -261,9 +261,6 @@ class FakeSwiftClient:
class BackupOpt1:
def __init__(self):
fakeclient = FakeSwiftClient()
fakeconnector = fakeclient.client()
fakeswclient = fakeconnector.Connection()
self.dereference_symlink = 'none'
self.mysql_conf = '/tmp/freezer-test-conf-file'
self.backup_media = 'fs'
@ -297,7 +294,6 @@ class BackupOpt1:
self.time_stamp = 123456789
self.container = 'test-container'
self.work_dir = '/tmp'
self.sw_connector = fakeswclient
self.max_level = '20'
self.encrypt_pass_file = '/dev/random'
self.always_level = '20'

1
freezer/tests/integration/common.py
View File

@ -297,4 +297,3 @@ class TestFS(unittest.TestCase):
result = execute_freezerc(restore_args)
self.assertIsNotNone(result)
self.assertTreesMatch()
return True

25
freezer/winutils.py
View File

@ -59,31 +59,6 @@ def use_shadow(to_backup, windows_volume):
.format(windows_volume))
def stop_sql_server(sql_server_instance):
""" Stop a SQL Server instance to perform the backup of the db files """
logging.info('[*] Stopping SQL Server for backup')
with DisableFileSystemRedirection():
cmd = 'net stop "SQL Server ({0})"'\
.format(sql_server_instance)
(out, err) = create_subprocess(cmd)
if err != '':
raise Exception('[*] Error while stopping SQL Server,'
', error {0}'.format(err))
def start_sql_server(sql_server_instance):
""" Start the SQL Server instance after the backup is completed """
with DisableFileSystemRedirection():
cmd = 'net start "SQL Server ({0})"'.format(sql_server_instance)
(out, err) = create_subprocess(cmd)
if err != '':
raise Exception('[*] Error while starting SQL Server'
', error {0}'.format(err))
logging.info('[*] SQL Server back to normal')
def save_environment(home):
"""Read the environment from the terminal where the scheduler is
initialized and save the environment variables to be reused within the

7
tests/unit/test_job.py
View File

@ -18,16 +18,13 @@ limitations under the License.
from freezer.tests.commons import *
from freezer.job import ExecJob
from freezer import backup
from freezer.job import Job, InfoJob, AdminJob, BackupJob
from mock import patch, Mock
import unittest
class TestJob(unittest.TestCase):
def test_execute(self):
opt = BackupOpt1()
job = Job(opt, opt.storage)
@ -51,7 +48,7 @@ class TestBackupJob(TestJob):
def test_execute_backup_fs_no_incremental_and_backup_level_raise(self):
backup_opt = BackupOpt1()
backup_opt.mode = 'fs'
backup_opt.mode = 'default'
backup_opt.no_incremental = True
job = BackupJob(backup_opt, backup_opt.storage)
self.assertRaises(Exception, job.execute)
@ -67,7 +64,7 @@ class TestBackupJob(TestJob):
class TestAdminJob(TestJob):
def test_execute(self):
backup_opt = BackupOpt1()
job = AdminJob(backup_opt, backup_opt.storage).execute()
AdminJob(backup_opt, backup_opt.storage).execute()
class TestExecJob(TestJob):

35
tests/unit/test_lvm.py
View File

@ -161,41 +161,6 @@ class Test_lvm_snap(unittest.TestCase):
self.assertTrue(lvm.lvm_snap(backup_opt))
@patch('freezer.lvm.subprocess.Popen')
@patch('freezer.lvm.utils.get_vol_fs_type')
@patch('freezer.lvm.get_lvm_info')
@patch('freezer.lvm.utils.create_dir')
def test_mysql_mode_locks_unlocks_tables(self, mock_create_dir, mock_get_lvm_info, mock_get_vol_fs_type, mock_popen):
mock_get_vol_fs_type.return_value = 'xfs'
mock_get_lvm_info.return_value = {
'volgroup': 'lvm_volgroup',
'srcvol': 'lvm_device',
'snap_path': 'snap_path'}
mock_process = Mock()
mock_process.communicate.return_value = '', ''
mock_process.returncode = 0
mock_popen.return_value = mock_process
backup_opt = Mock()
backup_opt.snapshot = True
backup_opt.lvm_auto_snap = ''
backup_opt.path_to_backup = '/just/a/path'
backup_opt.lvm_dirmount = '/var/mountpoint'
backup_opt.lvm_snapperm = 'ro'
backup_opt.mode = 'mysql'
backup_opt.mysql_db_inst = Mock()
mock_cursor = Mock()
backup_opt.mysql_db_inst.cursor.return_value = mock_cursor
self.assertTrue(lvm.lvm_snap(backup_opt))
first_call = call('FLUSH TABLES WITH READ LOCK')
second_call = call('UNLOCK TABLES')
self.assertEquals(first_call, mock_cursor.execute.call_args_list[0])
self.assertEquals(second_call, mock_cursor.execute.call_args_list[1])
@patch('freezer.lvm.lvm_snap_remove')
@patch('freezer.lvm.subprocess.Popen')
@patch('freezer.lvm.utils.get_vol_fs_type')