diff --git a/freezer/backup.py b/freezer/backup.py index 94b1f459..e78eb7ba 100644 --- a/freezer/backup.py +++ b/freezer/backup.py @@ -18,7 +18,6 @@ Freezer Backup modes related functions import os import time -from freezer import config from freezer import lvm from freezer import utils from freezer import vss @@ -31,93 +30,6 @@ logging = log.getLogger(__name__) home = os.path.expanduser("~") -def backup_mode_sql_server(backup_opt_dict, storage): - """ - Execute a SQL Server DB backup. Currently only backups with shadow - copy are supported. This mean, as soon as the shadow copy is created - the db writes will be blocked and a checkpoint will be created, as soon - as the backup finish the db will be unlocked and the backup will be - uploaded. A sql_server.conf_file is required for this operation. - """ - with open(backup_opt_dict.sql_server_conf, 'r') as sql_conf_file_fd: - parsed_config = config.ini_parse(sql_conf_file_fd.read()) - sql_server_instance = parsed_config["instance"] - # Dirty hack - please remove any modification of backup_opt_dict - backup_opt_dict.sql_server_instance = sql_server_instance - try: - winutils.stop_sql_server(sql_server_instance) - return backup(backup_opt_dict, storage, backup_opt_dict.engine) - finally: - if not backup_opt_dict.snapshot: - # if snapshot is false, wait until the backup is complete - # to start sql server again - winutils.start_sql_server(sql_server_instance) - - -def backup_mode_mysql(backup_opt_dict, storage): - """ - Execute a MySQL DB backup. currently only backup with lvm snapshots - are supported. This mean, just before the lvm snap vol is created, - the db tables will be flushed and locked for read, then the lvm create - command will be executed and after that, the table will be unlocked and - the backup will be executed. It is important to have the available in - backup_args.mysql_conf the file where the database host, name, user, - password and port are set. - """ - - try: - import pymysql as MySQLdb - except ImportError: - raise ImportError('Please install PyMySQL module') - - if not backup_opt_dict.mysql_conf: - raise ValueError('MySQL: please provide a valid config file') - with open(backup_opt_dict.mysql_conf, 'r') as mysql_file_fd: - parsed_config = config.ini_parse(mysql_file_fd.read()) - - # Initialize the DB object and connect to the db according to - # the db mysql backup file config - try: - backup_opt_dict.mysql_db_inst = MySQLdb.connect( - host=parsed_config.get("host", False), - port=int(parsed_config.get("port", 3306)), - user=parsed_config.get("user", False), - passwd=parsed_config.get("password", False)) - except Exception as error: - raise Exception('[*] MySQL: {0}'.format(error)) - - # Execute backup - return backup(backup_opt_dict, storage, backup_opt_dict.engine) - - -def backup_mode_mongo(backup_opt_dict, storage): - """ - Execute the necessary tasks for file system backup mode - """ - - try: - import pymongo - except ImportError: - raise ImportError('please install pymongo module') - - logging.info('[*] MongoDB backup is being executed...') - logging.info('[*] Checking is the localhost is Master/Primary...') - mongodb_port = '27017' - local_hostname = backup_opt_dict.hostname - db_host_port = '{0}:{1}'.format(local_hostname, mongodb_port) - mongo_client = pymongo.MongoClient(db_host_port) - master_dict = dict(mongo_client.admin.command("isMaster")) - mongo_me = master_dict['me'] - mongo_primary = master_dict['primary'] - - if mongo_me == mongo_primary: - return backup(backup_opt_dict, storage, backup_opt_dict.engine) - else: - logging.warning('[*] localhost {0} is not Master/Primary,\ - exiting...'.format(local_hostname)) - return None - - class BackupOs: def __init__(self, client_manager, container, storage): @@ -225,16 +137,9 @@ def snapshot_create(backup_opt_dict): backup_opt_dict.path_to_backup = winutils.use_shadow( backup_opt_dict.path_to_backup, backup_opt_dict.windows_volume) - - # execute this after the snapshot creation - if backup_opt_dict.mode == 'sqlserver': - winutils.start_sql_server(backup_opt_dict.sql_server_instance) - return True return False - else: - return lvm.lvm_snap(backup_opt_dict) @@ -247,7 +152,7 @@ def snapshot_remove(backup_opt_dict, shadow, windows_volume): lvm.lvm_snap_remove(backup_opt_dict) -def backup(backup_opt_dict, storage, engine): +def backup(backup_opt_dict, storage, engine, app_mode): """ :param backup_opt_dict: @@ -255,6 +160,7 @@ def backup(backup_opt_dict, storage, engine): :type storage: freezer.storage.base.Storage :param engine: Backup Engine :type engine: freezer.engine.engine.BackupEngine + :type app_mode: freezer.mode.mode.Mode :return: """ backup_media = backup_opt_dict.backup_media @@ -263,8 +169,10 @@ def backup(backup_opt_dict, storage, engine): backup_opt_dict.time_stamp = time_stamp if backup_media == 'fs': - + app_mode.prepare() snapshot_taken = snapshot_create(backup_opt_dict) + if snapshot_taken: + app_mode.release() try: filepath = '.' chdir_path = os.path.expanduser( @@ -285,6 +193,7 @@ def backup(backup_opt_dict, storage, engine): return backup_instance finally: # whether an error occurred or not, remove the snapshot anyway + app_mode.release() if snapshot_taken: snapshot_remove(backup_opt_dict, backup_opt_dict.shadow, backup_opt_dict.windows_volume) diff --git a/freezer/common/config.py b/freezer/common/config.py index 34c4087b..98216586 100644 --- a/freezer/common/config.py +++ b/freezer/common/config.py @@ -59,7 +59,7 @@ DEFAULT_PARAMS = { 'upload_limit': -1, 'always_level': False, 'version': False, 'dry_run': False, 'lvm_snapsize': DEFAULT_LVM_SNAPSIZE, 'restore_abs_path': False, 'log_file': None, 'log_level': "info", - 'mode': 'fs', 'action': 'backup', 'shadow': '', 'shadow_path': '', + 'mode': 'default', 'action': 'backup', 'shadow': '', 'shadow_path': '', 'windows_volume': '', 'command': None, 'metadata_out': False, 'storage': 'swift', 'ssh_key': '', 'ssh_username': '', 'ssh_host': '', 'ssh_port': DEFAULT_SSH_PORT, 'compression': 'gzip' diff --git a/freezer/job.py b/freezer/job.py index 66e2a4b2..4677018b 100644 --- a/freezer/job.py +++ b/freezer/job.py @@ -16,6 +16,7 @@ limitations under the License. """ import datetime +from oslo_utils import importutils import sys import time @@ -78,21 +79,13 @@ class BackupJob(Job): logging.error('Error while sync exec: {0}'.format(err)) except Exception as error: logging.error('Error while sync exec: {0}'.format(error)) - - if self.conf.mode == 'fs': - backup_instance = \ - backup.backup(self.conf, self.storage, self.engine) - elif self.conf.mode == 'mongo': - backup_instance = \ - backup.backup_mode_mongo(self.conf, self.storage) - elif self.conf.mode == 'mysql': - backup_instance = \ - backup.backup_mode_mysql(self.conf, self.storage) - elif self.conf.mode == 'sqlserver': - backup_instance = \ - backup.backup_mode_sql_server(self.conf, self.storage) - else: - raise ValueError('Please provide a valid backup mode') + if not self.conf.mode: + raise ValueError("Empty mode") + mod_name = 'freezer.mode.{0}.{1}'.format( + self.conf.mode, self.conf.mode.capitalize() + 'Mode') + app_mode = importutils.import_object(mod_name, self.conf) + backup_instance = backup.backup( + self.conf, self.storage, self.engine, app_mode) level = backup_instance.level if backup_instance else 0 @@ -133,7 +126,7 @@ class RestoreJob(Job): restore_timestamp) self.engine.restore(backup, restore_abs_path) - return + return {} res = restore.RestoreOs(conf.client_manager, conf.container) if conf.backup_media == 'nova': diff --git a/freezer/lvm.py b/freezer/lvm.py index 983999c8..4b4bdb16 100644 --- a/freezer/lvm.py +++ b/freezer/lvm.py @@ -126,27 +126,12 @@ def lvm_snap(backup_opt_dict): backup_opt_dict.lvm_snapname, backup_opt_dict.lvm_srcvol)) - # If backup mode is mysql, then the db will be flushed and read locked - # before the creation of the lvm snap - if backup_opt_dict.mode == 'mysql': - cursor = backup_opt_dict.mysql_db_inst.cursor() - cursor.execute('FLUSH TABLES WITH READ LOCK') - backup_opt_dict.mysql_db_inst.commit() - lvm_process = subprocess.Popen( lvm_create_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, executable=utils.find_executable('bash')) (lvm_out, lvm_err) = lvm_process.communicate() - # Unlock MySQL Tables if backup is == mysql - # regardless of the snapshot being taken or not - if backup_opt_dict.mode == 'mysql': - cursor.execute('UNLOCK TABLES') - backup_opt_dict.mysql_db_inst.commit() - cursor.close() - backup_opt_dict.mysql_db_inst.close() - if lvm_process.returncode: raise Exception('lvm snapshot creation error: {0}'.format(lvm_err)) diff --git a/freezer/mode/__init__.py b/freezer/mode/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freezer/mode/default.py b/freezer/mode/default.py new file mode 100644 index 00000000..2413747f --- /dev/null +++ b/freezer/mode/default.py @@ -0,0 +1,35 @@ +# (c) Copyright 2015,2016 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from freezer.mode import mode + + +class DefaultMode(mode.Mode): + + def __init__(self, conf): + pass + + @property + def name(self): + return "default" + + @property + def version(self): + return "1.0" + + def release(self): + pass + + def prepare(self): + pass diff --git a/freezer/mode/mode.py b/freezer/mode/mode.py new file mode 100644 index 00000000..b9e6def5 --- /dev/null +++ b/freezer/mode/mode.py @@ -0,0 +1,35 @@ +# (c) Copyright 2015,2016 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import abc +import six + + +@six.add_metaclass(abc.ABCMeta) +class Mode(object): + @abc.abstractproperty + def name(self): + pass + + @abc.abstractproperty + def version(self): + pass + + @abc.abstractmethod + def prepare(self): + pass + + @abc.abstractmethod + def release(self): + pass diff --git a/freezer/mode/mongo.py b/freezer/mode/mongo.py new file mode 100644 index 00000000..30d31be9 --- /dev/null +++ b/freezer/mode/mongo.py @@ -0,0 +1,55 @@ +# (c) Copyright 2015,2016 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from freezer.mode import mode + + +class MongoDbMode(mode.Mode): + """ + Execute the necessary tasks for file system backup mode + """ + + @property + def name(self): + return "mongo" + + @property + def version(self): + return "1.0" + + def release(self): + pass + + def prepare(self): + pass + + def __init__(self, conf): + try: + import pymongo + except ImportError: + raise ImportError('please install pymongo module') + + logging.info('[*] MongoDB backup is being executed...') + logging.info('[*] Checking is the localhost is Master/Primary...') + # todo unhardcode this + mongodb_port = '27017' + local_hostname = conf.hostname + db_host_port = '{0}:{1}'.format(local_hostname, mongodb_port) + mongo_client = pymongo.MongoClient(db_host_port) + master_dict = dict(mongo_client.admin.command("isMaster")) + if master_dict['me'] != master_dict['primary']: + raise Exception('[*] localhost {0} is not Master/Primary,\ + exiting...'.format(local_hostname)) diff --git a/freezer/mode/mysql.py b/freezer/mode/mysql.py new file mode 100644 index 00000000..ee3848be --- /dev/null +++ b/freezer/mode/mysql.py @@ -0,0 +1,71 @@ +# (c) Copyright 2015,2016 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from freezer import config +from freezer.mode import mode + + +class MysqlMode(mode.Mode): + """ + Execute a MySQL DB backup. currently only backup with lvm snapshots + are supported. This mean, just before the lvm snap vol is created, + the db tables will be flushed and locked for read, then the lvm create + command will be executed and after that, the table will be unlocked and + the backup will be executed. It is important to have the available in + backup_args.mysql_conf the file where the database host, name, user, + password and port are set. + """ + + @property + def name(self): + return "mysql" + + @property + def version(self): + return "1.0" + + def release(self): + if not self.released: + self.released = True + self.cursor.execute('UNLOCK TABLES') + self.mysql_db_inst.commit() + self.cursor.close() + self.mysql_db_inst.close() + + def prepare(self): + self.released = False + self.cursor = self.mysql_db_inst.cursor() + self.cursor.execute('FLUSH TABLES WITH READ LOCK') + self.mysql_db_inst.commit() + + def __init__(self, conf): + try: + import pymysql as MySQLdb + except ImportError: + raise ImportError('Please install PyMySQL module') + + with open(conf.mysql_conf, 'r') as mysql_file_fd: + parsed_config = config.ini_parse(mysql_file_fd.read()) + # Initialize the DB object and connect to the db according to + # the db mysql backup file config + self.released = False + try: + self.mysql_db_inst = MySQLdb.connect( + host=parsed_config.get("host", False), + port=int(parsed_config.get("port", 3306)), + user=parsed_config.get("user", False), + passwd=parsed_config.get("password", False)) + self.cursor = None + except Exception as error: + raise Exception('[*] MySQL: {0}'.format(error)) diff --git a/freezer/mode/sqlserver.py b/freezer/mode/sqlserver.py new file mode 100644 index 00000000..10e2ac07 --- /dev/null +++ b/freezer/mode/sqlserver.py @@ -0,0 +1,77 @@ +# (c) Copyright 2015,2016 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from freezer import config +from freezer.mode import mode +from freezer import utils +from freezer import winutils + + +class SqlserverMode(mode.Mode): + """ + Execute a SQL Server DB backup. Currently only backups with shadow + copy are supported. This mean, as soon as the shadow copy is created + the db writes will be blocked and a checkpoint will be created, as soon + as the backup finish the db will be unlocked and the backup will be + uploaded. A sql_server.conf_file is required for this operation. + """ + def __init__(self, conf): + self.released = False + with open(conf.sql_server_conf, 'r') as sql_conf_file_fd: + self.sql_server_instance = \ + config.ini_parse(sql_conf_file_fd.read())["instance"] + + @property + def name(self): + return "sqlserver" + + @property + def version(self): + return "1.0" + + def stop_sql_server(self): + """ Stop a SQL Server instance to + perform the backup of the db files """ + + logging.info('[*] Stopping SQL Server for backup') + with winutils.DisableFileSystemRedirection(): + cmd = 'net stop "SQL Server ({0})"'\ + .format(self.sql_server_instance) + (out, err) = utils.create_subprocess(cmd) + if err != '': + raise Exception('[*] Error while stopping SQL Server,' + ', error {0}'.format(err)) + + def start_sql_server(self): + """ Start the SQL Server instance after the backup is completed """ + + with winutils.DisableFileSystemRedirection(): + cmd = 'net start "SQL Server ({0})"'.format( + self.sql_server_instance) + (out, err) = utils.create_subprocess(cmd) + if err != '': + raise Exception('[*] Error while starting SQL Server' + ', error {0}'.format(err)) + logging.info('[*] SQL Server back to normal') + + def prepare(self): + self.stop_sql_server() + self.released = False + + def release(self): + if not self.released: + self.released = True + self.start_sql_server() diff --git a/freezer/tests/commons.py b/freezer/tests/commons.py index 508bfc87..162c49a1 100644 --- a/freezer/tests/commons.py +++ b/freezer/tests/commons.py @@ -261,9 +261,6 @@ class FakeSwiftClient: class BackupOpt1: def __init__(self): - fakeclient = FakeSwiftClient() - fakeconnector = fakeclient.client() - fakeswclient = fakeconnector.Connection() self.dereference_symlink = 'none' self.mysql_conf = '/tmp/freezer-test-conf-file' self.backup_media = 'fs' @@ -297,7 +294,6 @@ class BackupOpt1: self.time_stamp = 123456789 self.container = 'test-container' self.work_dir = '/tmp' - self.sw_connector = fakeswclient self.max_level = '20' self.encrypt_pass_file = '/dev/random' self.always_level = '20' diff --git a/freezer/tests/integration/common.py b/freezer/tests/integration/common.py index f3d7fc73..c9ec4b27 100644 --- a/freezer/tests/integration/common.py +++ b/freezer/tests/integration/common.py @@ -297,4 +297,3 @@ class TestFS(unittest.TestCase): result = execute_freezerc(restore_args) self.assertIsNotNone(result) self.assertTreesMatch() - return True diff --git a/freezer/winutils.py b/freezer/winutils.py index 82151967..21cb1b34 100644 --- a/freezer/winutils.py +++ b/freezer/winutils.py @@ -59,31 +59,6 @@ def use_shadow(to_backup, windows_volume): .format(windows_volume)) -def stop_sql_server(sql_server_instance): - """ Stop a SQL Server instance to perform the backup of the db files """ - - logging.info('[*] Stopping SQL Server for backup') - with DisableFileSystemRedirection(): - cmd = 'net stop "SQL Server ({0})"'\ - .format(sql_server_instance) - (out, err) = create_subprocess(cmd) - if err != '': - raise Exception('[*] Error while stopping SQL Server,' - ', error {0}'.format(err)) - - -def start_sql_server(sql_server_instance): - """ Start the SQL Server instance after the backup is completed """ - - with DisableFileSystemRedirection(): - cmd = 'net start "SQL Server ({0})"'.format(sql_server_instance) - (out, err) = create_subprocess(cmd) - if err != '': - raise Exception('[*] Error while starting SQL Server' - ', error {0}'.format(err)) - logging.info('[*] SQL Server back to normal') - - def save_environment(home): """Read the environment from the terminal where the scheduler is initialized and save the environment variables to be reused within the diff --git a/tests/unit/test_job.py b/tests/unit/test_job.py index 8aeef420..eacea820 100644 --- a/tests/unit/test_job.py +++ b/tests/unit/test_job.py @@ -18,16 +18,13 @@ limitations under the License. from freezer.tests.commons import * from freezer.job import ExecJob -from freezer import backup from freezer.job import Job, InfoJob, AdminJob, BackupJob from mock import patch, Mock import unittest - class TestJob(unittest.TestCase): - def test_execute(self): opt = BackupOpt1() job = Job(opt, opt.storage) @@ -51,7 +48,7 @@ class TestBackupJob(TestJob): def test_execute_backup_fs_no_incremental_and_backup_level_raise(self): backup_opt = BackupOpt1() - backup_opt.mode = 'fs' + backup_opt.mode = 'default' backup_opt.no_incremental = True job = BackupJob(backup_opt, backup_opt.storage) self.assertRaises(Exception, job.execute) @@ -67,7 +64,7 @@ class TestBackupJob(TestJob): class TestAdminJob(TestJob): def test_execute(self): backup_opt = BackupOpt1() - job = AdminJob(backup_opt, backup_opt.storage).execute() + AdminJob(backup_opt, backup_opt.storage).execute() class TestExecJob(TestJob): diff --git a/tests/unit/test_lvm.py b/tests/unit/test_lvm.py index be28bedd..58778b3b 100644 --- a/tests/unit/test_lvm.py +++ b/tests/unit/test_lvm.py @@ -161,41 +161,6 @@ class Test_lvm_snap(unittest.TestCase): self.assertTrue(lvm.lvm_snap(backup_opt)) - - @patch('freezer.lvm.subprocess.Popen') - @patch('freezer.lvm.utils.get_vol_fs_type') - @patch('freezer.lvm.get_lvm_info') - @patch('freezer.lvm.utils.create_dir') - def test_mysql_mode_locks_unlocks_tables(self, mock_create_dir, mock_get_lvm_info, mock_get_vol_fs_type, mock_popen): - mock_get_vol_fs_type.return_value = 'xfs' - mock_get_lvm_info.return_value = { - 'volgroup': 'lvm_volgroup', - 'srcvol': 'lvm_device', - 'snap_path': 'snap_path'} - mock_process = Mock() - mock_process.communicate.return_value = '', '' - mock_process.returncode = 0 - mock_popen.return_value = mock_process - - backup_opt = Mock() - backup_opt.snapshot = True - backup_opt.lvm_auto_snap = '' - backup_opt.path_to_backup = '/just/a/path' - backup_opt.lvm_dirmount = '/var/mountpoint' - backup_opt.lvm_snapperm = 'ro' - backup_opt.mode = 'mysql' - backup_opt.mysql_db_inst = Mock() - mock_cursor = Mock() - backup_opt.mysql_db_inst.cursor.return_value = mock_cursor - - self.assertTrue(lvm.lvm_snap(backup_opt)) - - first_call = call('FLUSH TABLES WITH READ LOCK') - second_call = call('UNLOCK TABLES') - self.assertEquals(first_call, mock_cursor.execute.call_args_list[0]) - self.assertEquals(second_call, mock_cursor.execute.call_args_list[1]) - - @patch('freezer.lvm.lvm_snap_remove') @patch('freezer.lvm.subprocess.Popen') @patch('freezer.lvm.utils.get_vol_fs_type')