Refactoring of mysql, msserver and mongo code.

Implements blueprint: application-hook-refactoring

Change-Id: I6df7ae22ec17479dc926cc4e764091487544b5fb

freezer/backup.py

@@ -18,7 +18,6 @@ Freezer Backup modes related functions
 import os
 import time
 
-from freezer import config
 from freezer import lvm
 from freezer import utils
 from freezer import vss
@@ -31,93 +30,6 @@ logging = log.getLogger(__name__)
 home = os.path.expanduser("~")
 
 
-def backup_mode_sql_server(backup_opt_dict, storage):
-    """
-    Execute a SQL Server DB backup. Currently only backups with shadow
-    copy are supported. This mean, as soon as the shadow copy is created
-    the db writes will be blocked and a checkpoint will be created, as soon
-    as the backup finish the db will be unlocked and the backup will be
-    uploaded. A sql_server.conf_file is required for this operation.
-    """
-    with open(backup_opt_dict.sql_server_conf, 'r') as sql_conf_file_fd:
-        parsed_config = config.ini_parse(sql_conf_file_fd.read())
-    sql_server_instance = parsed_config["instance"]
-    # Dirty hack - please remove any modification of backup_opt_dict
-    backup_opt_dict.sql_server_instance = sql_server_instance
-    try:
-        winutils.stop_sql_server(sql_server_instance)
-        return backup(backup_opt_dict, storage, backup_opt_dict.engine)
-    finally:
-        if not backup_opt_dict.snapshot:
-            # if snapshot is false, wait until the backup is complete
-            # to start sql server again
-            winutils.start_sql_server(sql_server_instance)
-
-
-def backup_mode_mysql(backup_opt_dict, storage):
-    """
-    Execute a MySQL DB backup. currently only backup with lvm snapshots
-    are supported. This mean, just before the lvm snap vol is created,
-    the db tables will be flushed and locked for read, then the lvm create
-    command will be executed and after that, the table will be unlocked and
-    the backup will be executed. It is important to have the available in
-    backup_args.mysql_conf the file where the database host, name, user,
-    password and port are set.
-    """
-
-    try:
-        import pymysql as MySQLdb
-    except ImportError:
-        raise ImportError('Please install PyMySQL module')
-
-    if not backup_opt_dict.mysql_conf:
-        raise ValueError('MySQL: please provide a valid config file')
-    with open(backup_opt_dict.mysql_conf, 'r') as mysql_file_fd:
-        parsed_config = config.ini_parse(mysql_file_fd.read())
-
-    # Initialize the DB object and connect to the db according to
-    # the db mysql backup file config
-    try:
-        backup_opt_dict.mysql_db_inst = MySQLdb.connect(
-            host=parsed_config.get("host", False),
-            port=int(parsed_config.get("port", 3306)),
-            user=parsed_config.get("user", False),
-            passwd=parsed_config.get("password", False))
-    except Exception as error:
-        raise Exception('[*] MySQL: {0}'.format(error))
-
-    # Execute backup
-    return backup(backup_opt_dict, storage, backup_opt_dict.engine)
-
-
-def backup_mode_mongo(backup_opt_dict, storage):
-    """
-    Execute the necessary tasks for file system backup mode
-    """
-
-    try:
-        import pymongo
-    except ImportError:
-        raise ImportError('please install pymongo module')
-
-    logging.info('[*] MongoDB backup is being executed...')
-    logging.info('[*] Checking is the localhost is Master/Primary...')
-    mongodb_port = '27017'
-    local_hostname = backup_opt_dict.hostname
-    db_host_port = '{0}:{1}'.format(local_hostname, mongodb_port)
-    mongo_client = pymongo.MongoClient(db_host_port)
-    master_dict = dict(mongo_client.admin.command("isMaster"))
-    mongo_me = master_dict['me']
-    mongo_primary = master_dict['primary']
-
-    if mongo_me == mongo_primary:
-        return backup(backup_opt_dict, storage, backup_opt_dict.engine)
-    else:
-        logging.warning('[*] localhost {0} is not Master/Primary,\
-        exiting...'.format(local_hostname))
-        return None
-
-
 class BackupOs:
 
     def __init__(self, client_manager, container, storage):
@@ -225,16 +137,9 @@ def snapshot_create(backup_opt_dict):
             backup_opt_dict.path_to_backup = winutils.use_shadow(
                 backup_opt_dict.path_to_backup,
                 backup_opt_dict.windows_volume)
-
-            # execute this after the snapshot creation
-            if backup_opt_dict.mode == 'sqlserver':
-                winutils.start_sql_server(backup_opt_dict.sql_server_instance)
-
             return True
         return False
-
     else:
-
         return lvm.lvm_snap(backup_opt_dict)
 
 
@@ -247,7 +152,7 @@ def snapshot_remove(backup_opt_dict, shadow, windows_volume):
         lvm.lvm_snap_remove(backup_opt_dict)
 
 
-def backup(backup_opt_dict, storage, engine):
+def backup(backup_opt_dict, storage, engine, app_mode):
     """
 
     :param backup_opt_dict:
@@ -255,6 +160,7 @@ def backup(backup_opt_dict, storage, engine):
     :type storage: freezer.storage.base.Storage
     :param engine: Backup Engine
     :type engine: freezer.engine.engine.BackupEngine
+    :type app_mode: freezer.mode.mode.Mode
     :return:
     """
     backup_media = backup_opt_dict.backup_media
@@ -263,8 +169,10 @@ def backup(backup_opt_dict, storage, engine):
     backup_opt_dict.time_stamp = time_stamp
 
     if backup_media == 'fs':
-
+        app_mode.prepare()
         snapshot_taken = snapshot_create(backup_opt_dict)
+        if snapshot_taken:
+            app_mode.release()
         try:
             filepath = '.'
             chdir_path = os.path.expanduser(
@@ -285,6 +193,7 @@ def backup(backup_opt_dict, storage, engine):
             return backup_instance
         finally:
             # whether an error occurred or not, remove the snapshot anyway
+            app_mode.release()
             if snapshot_taken:
                 snapshot_remove(backup_opt_dict, backup_opt_dict.shadow,
                                 backup_opt_dict.windows_volume)

freezer/common/config.py

@@ -59,7 +59,7 @@ DEFAULT_PARAMS = {
     'upload_limit': -1, 'always_level': False, 'version': False,
     'dry_run': False, 'lvm_snapsize': DEFAULT_LVM_SNAPSIZE,
     'restore_abs_path': False, 'log_file': None, 'log_level': "info",
-    'mode': 'fs', 'action': 'backup', 'shadow': '', 'shadow_path': '',
+    'mode': 'default', 'action': 'backup', 'shadow': '', 'shadow_path': '',
     'windows_volume': '', 'command': None, 'metadata_out': False,
     'storage': 'swift', 'ssh_key': '', 'ssh_username': '', 'ssh_host': '',
     'ssh_port': DEFAULT_SSH_PORT, 'compression': 'gzip'

freezer/job.py

@@ -16,6 +16,7 @@ limitations under the License.
 """
 
 import datetime
+from oslo_utils import importutils
 import sys
 import time
 
@@ -78,21 +79,13 @@ class BackupJob(Job):
                 logging.error('Error while sync exec: {0}'.format(err))
         except Exception as error:
             logging.error('Error while sync exec: {0}'.format(error))
-
+        if not self.conf.mode:
-        if self.conf.mode == 'fs':
+            raise ValueError("Empty mode")
-            backup_instance = \
+        mod_name = 'freezer.mode.{0}.{1}'.format(
-                backup.backup(self.conf, self.storage, self.engine)
+            self.conf.mode, self.conf.mode.capitalize() + 'Mode')
-        elif self.conf.mode == 'mongo':
+        app_mode = importutils.import_object(mod_name, self.conf)
-            backup_instance = \
+        backup_instance = backup.backup(
-                backup.backup_mode_mongo(self.conf, self.storage)
+            self.conf, self.storage, self.engine, app_mode)
-        elif self.conf.mode == 'mysql':
-            backup_instance = \
-                backup.backup_mode_mysql(self.conf, self.storage)
-        elif self.conf.mode == 'sqlserver':
-            backup_instance = \
-                backup.backup_mode_sql_server(self.conf, self.storage)
-        else:
-            raise ValueError('Please provide a valid backup mode')
 
         level = backup_instance.level if backup_instance else 0
 
@@ -133,7 +126,7 @@ class RestoreJob(Job):
                                            restore_timestamp)
 
             self.engine.restore(backup, restore_abs_path)
-            return
+            return {}
 
         res = restore.RestoreOs(conf.client_manager, conf.container)
         if conf.backup_media == 'nova':

freezer/lvm.py

@@ -126,27 +126,12 @@ def lvm_snap(backup_opt_dict):
             backup_opt_dict.lvm_snapname,
             backup_opt_dict.lvm_srcvol))
 
-    # If backup mode is mysql, then the db will be flushed and read locked
-    # before the creation of the lvm snap
-    if backup_opt_dict.mode == 'mysql':
-        cursor = backup_opt_dict.mysql_db_inst.cursor()
-        cursor.execute('FLUSH TABLES WITH READ LOCK')
-        backup_opt_dict.mysql_db_inst.commit()
-
     lvm_process = subprocess.Popen(
         lvm_create_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
         stderr=subprocess.PIPE, shell=True,
         executable=utils.find_executable('bash'))
     (lvm_out, lvm_err) = lvm_process.communicate()
 
-    # Unlock MySQL Tables if backup is == mysql
-    # regardless of the snapshot being taken or not
-    if backup_opt_dict.mode == 'mysql':
-        cursor.execute('UNLOCK TABLES')
-        backup_opt_dict.mysql_db_inst.commit()
-        cursor.close()
-        backup_opt_dict.mysql_db_inst.close()
-
     if lvm_process.returncode:
         raise Exception('lvm snapshot creation error: {0}'.format(lvm_err))
 

freezer/mode/default.py

@@ -0,0 +1,35 @@
+# (c) Copyright 2015,2016 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from freezer.mode import mode
+
+
+class DefaultMode(mode.Mode):
+
+    def __init__(self, conf):
+        pass
+
+    @property
+    def name(self):
+        return "default"
+
+    @property
+    def version(self):
+        return "1.0"
+
+    def release(self):
+        pass
+
+    def prepare(self):
+        pass

freezer/mode/mode.py

@@ -0,0 +1,35 @@
+# (c) Copyright 2015,2016 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import abc
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class Mode(object):
+    @abc.abstractproperty
+    def name(self):
+        pass
+
+    @abc.abstractproperty
+    def version(self):
+        pass
+
+    @abc.abstractmethod
+    def prepare(self):
+        pass
+
+    @abc.abstractmethod
+    def release(self):
+        pass

freezer/mode/mongo.py

@@ -0,0 +1,55 @@
+# (c) Copyright 2015,2016 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from freezer.mode import mode
+
+
+class MongoDbMode(mode.Mode):
+    """
+    Execute the necessary tasks for file system backup mode
+    """
+
+    @property
+    def name(self):
+        return "mongo"
+
+    @property
+    def version(self):
+        return "1.0"
+
+    def release(self):
+        pass
+
+    def prepare(self):
+        pass
+
+    def __init__(self, conf):
+        try:
+            import pymongo
+        except ImportError:
+            raise ImportError('please install pymongo module')
+
+        logging.info('[*] MongoDB backup is being executed...')
+        logging.info('[*] Checking is the localhost is Master/Primary...')
+        # todo unhardcode this
+        mongodb_port = '27017'
+        local_hostname = conf.hostname
+        db_host_port = '{0}:{1}'.format(local_hostname, mongodb_port)
+        mongo_client = pymongo.MongoClient(db_host_port)
+        master_dict = dict(mongo_client.admin.command("isMaster"))
+        if master_dict['me'] != master_dict['primary']:
+            raise Exception('[*] localhost {0} is not Master/Primary,\
+                exiting...'.format(local_hostname))

freezer/mode/mysql.py

@@ -0,0 +1,71 @@
+# (c) Copyright 2015,2016 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from freezer import config
+from freezer.mode import mode
+
+
+class MysqlMode(mode.Mode):
+    """
+    Execute a MySQL DB backup. currently only backup with lvm snapshots
+    are supported. This mean, just before the lvm snap vol is created,
+    the db tables will be flushed and locked for read, then the lvm create
+    command will be executed and after that, the table will be unlocked and
+    the backup will be executed. It is important to have the available in
+    backup_args.mysql_conf the file where the database host, name, user,
+    password and port are set.
+    """
+
+    @property
+    def name(self):
+        return "mysql"
+
+    @property
+    def version(self):
+        return "1.0"
+
+    def release(self):
+        if not self.released:
+            self.released = True
+            self.cursor.execute('UNLOCK TABLES')
+            self.mysql_db_inst.commit()
+            self.cursor.close()
+            self.mysql_db_inst.close()
+
+    def prepare(self):
+        self.released = False
+        self.cursor = self.mysql_db_inst.cursor()
+        self.cursor.execute('FLUSH TABLES WITH READ LOCK')
+        self.mysql_db_inst.commit()
+
+    def __init__(self, conf):
+        try:
+            import pymysql as MySQLdb
+        except ImportError:
+            raise ImportError('Please install PyMySQL module')
+
+        with open(conf.mysql_conf, 'r') as mysql_file_fd:
+            parsed_config = config.ini_parse(mysql_file_fd.read())
+        # Initialize the DB object and connect to the db according to
+        # the db mysql backup file config
+        self.released = False
+        try:
+            self.mysql_db_inst = MySQLdb.connect(
+                host=parsed_config.get("host", False),
+                port=int(parsed_config.get("port", 3306)),
+                user=parsed_config.get("user", False),
+                passwd=parsed_config.get("password", False))
+            self.cursor = None
+        except Exception as error:
+            raise Exception('[*] MySQL: {0}'.format(error))

freezer/mode/sqlserver.py

@@ -0,0 +1,77 @@
+# (c) Copyright 2015,2016 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from freezer import config
+from freezer.mode import mode
+from freezer import utils
+from freezer import winutils
+
+
+class SqlserverMode(mode.Mode):
+    """
+    Execute a SQL Server DB backup. Currently only backups with shadow
+    copy are supported. This mean, as soon as the shadow copy is created
+    the db writes will be blocked and a checkpoint will be created, as soon
+    as the backup finish the db will be unlocked and the backup will be
+    uploaded. A sql_server.conf_file is required for this operation.
+    """
+    def __init__(self, conf):
+        self.released = False
+        with open(conf.sql_server_conf, 'r') as sql_conf_file_fd:
+            self.sql_server_instance = \
+                config.ini_parse(sql_conf_file_fd.read())["instance"]
+
+    @property
+    def name(self):
+        return "sqlserver"
+
+    @property
+    def version(self):
+        return "1.0"
+
+    def stop_sql_server(self):
+        """ Stop a SQL Server instance to
+        perform the backup of the db files """
+
+        logging.info('[*] Stopping SQL Server for backup')
+        with winutils.DisableFileSystemRedirection():
+            cmd = 'net stop "SQL Server ({0})"'\
+                .format(self.sql_server_instance)
+            (out, err) = utils.create_subprocess(cmd)
+            if err != '':
+                raise Exception('[*] Error while stopping SQL Server,'
+                                ', error {0}'.format(err))
+
+    def start_sql_server(self):
+        """ Start the SQL Server instance after the backup is completed """
+
+        with winutils.DisableFileSystemRedirection():
+            cmd = 'net start "SQL Server ({0})"'.format(
+                self.sql_server_instance)
+            (out, err) = utils.create_subprocess(cmd)
+            if err != '':
+                raise Exception('[*] Error while starting SQL Server'
+                                ', error {0}'.format(err))
+            logging.info('[*] SQL Server back to normal')
+
+    def prepare(self):
+        self.stop_sql_server()
+        self.released = False
+
+    def release(self):
+        if not self.released:
+            self.released = True
+            self.start_sql_server()

freezer/tests/commons.py

@@ -261,9 +261,6 @@ class FakeSwiftClient:
 class BackupOpt1:
 
     def __init__(self):
-        fakeclient = FakeSwiftClient()
-        fakeconnector = fakeclient.client()
-        fakeswclient = fakeconnector.Connection()
         self.dereference_symlink = 'none'
         self.mysql_conf = '/tmp/freezer-test-conf-file'
         self.backup_media = 'fs'
@@ -297,7 +294,6 @@ class BackupOpt1:
         self.time_stamp = 123456789
         self.container = 'test-container'
         self.work_dir = '/tmp'
-        self.sw_connector = fakeswclient
         self.max_level = '20'
         self.encrypt_pass_file = '/dev/random'
         self.always_level = '20'

freezer/tests/integration/common.py

@@ -297,4 +297,3 @@ class TestFS(unittest.TestCase):
         result = execute_freezerc(restore_args)
         self.assertIsNotNone(result)
         self.assertTreesMatch()
-        return True

freezer/winutils.py

@@ -59,31 +59,6 @@ def use_shadow(to_backup, windows_volume):
                              .format(windows_volume))
 
 
-def stop_sql_server(sql_server_instance):
-    """ Stop a SQL Server instance to perform the backup of the db files """
-
-    logging.info('[*] Stopping SQL Server for backup')
-    with DisableFileSystemRedirection():
-        cmd = 'net stop "SQL Server ({0})"'\
-            .format(sql_server_instance)
-        (out, err) = create_subprocess(cmd)
-        if err != '':
-            raise Exception('[*] Error while stopping SQL Server,'
-                            ', error {0}'.format(err))
-
-
-def start_sql_server(sql_server_instance):
-    """ Start the SQL Server instance after the backup is completed """
-
-    with DisableFileSystemRedirection():
-        cmd = 'net start "SQL Server ({0})"'.format(sql_server_instance)
-        (out, err) = create_subprocess(cmd)
-        if err != '':
-            raise Exception('[*] Error while starting SQL Server'
-                            ', error {0}'.format(err))
-        logging.info('[*] SQL Server back to normal')
-
-
 def save_environment(home):
     """Read the environment from the terminal where the scheduler is
     initialized and save the environment variables to be reused within the

tests/unit/test_job.py

@@ -18,16 +18,13 @@ limitations under the License.
 
 from freezer.tests.commons import *
 from freezer.job import ExecJob
-from freezer import backup
 
 from freezer.job import Job, InfoJob, AdminJob, BackupJob
 from mock import patch, Mock
 import unittest
 
 
-
 class TestJob(unittest.TestCase):
-
     def test_execute(self):
         opt = BackupOpt1()
         job = Job(opt, opt.storage)
@@ -51,7 +48,7 @@ class TestBackupJob(TestJob):
 
     def test_execute_backup_fs_no_incremental_and_backup_level_raise(self):
         backup_opt = BackupOpt1()
-        backup_opt.mode = 'fs'
+        backup_opt.mode = 'default'
         backup_opt.no_incremental = True
         job = BackupJob(backup_opt, backup_opt.storage)
         self.assertRaises(Exception, job.execute)
@@ -67,7 +64,7 @@ class TestBackupJob(TestJob):
 class TestAdminJob(TestJob):
     def test_execute(self):
         backup_opt = BackupOpt1()
-        job = AdminJob(backup_opt, backup_opt.storage).execute()
+        AdminJob(backup_opt, backup_opt.storage).execute()
 
 
 class TestExecJob(TestJob):

tests/unit/test_lvm.py

@@ -161,41 +161,6 @@ class Test_lvm_snap(unittest.TestCase):
 
         self.assertTrue(lvm.lvm_snap(backup_opt))
 
-
-    @patch('freezer.lvm.subprocess.Popen')
-    @patch('freezer.lvm.utils.get_vol_fs_type')
-    @patch('freezer.lvm.get_lvm_info')
-    @patch('freezer.lvm.utils.create_dir')
-    def test_mysql_mode_locks_unlocks_tables(self, mock_create_dir, mock_get_lvm_info, mock_get_vol_fs_type, mock_popen):
-        mock_get_vol_fs_type.return_value = 'xfs'
-        mock_get_lvm_info.return_value = {
-            'volgroup': 'lvm_volgroup',
-            'srcvol': 'lvm_device',
-            'snap_path': 'snap_path'}
-        mock_process = Mock()
-        mock_process.communicate.return_value = '', ''
-        mock_process.returncode = 0
-        mock_popen.return_value = mock_process
-
-        backup_opt = Mock()
-        backup_opt.snapshot = True
-        backup_opt.lvm_auto_snap = ''
-        backup_opt.path_to_backup = '/just/a/path'
-        backup_opt.lvm_dirmount = '/var/mountpoint'
-        backup_opt.lvm_snapperm = 'ro'
-        backup_opt.mode = 'mysql'
-        backup_opt.mysql_db_inst = Mock()
-        mock_cursor = Mock()
-        backup_opt.mysql_db_inst.cursor.return_value = mock_cursor
-
-        self.assertTrue(lvm.lvm_snap(backup_opt))
-
-        first_call = call('FLUSH TABLES WITH READ LOCK')
-        second_call = call('UNLOCK TABLES')
-        self.assertEquals(first_call, mock_cursor.execute.call_args_list[0])
-        self.assertEquals(second_call, mock_cursor.execute.call_args_list[1])
-
-
     @patch('freezer.lvm.lvm_snap_remove')
     @patch('freezer.lvm.subprocess.Popen')
     @patch('freezer.lvm.utils.get_vol_fs_type')