Reduce SQL Server downtime and Snapshot option on windows

CLI argument for snapshots on windows (vssadmin)
Reduce downtime with SQL Server if a snapshot is available

Change-Id: Iaae4d028cd19e1732925ec5bd86665b839528cd3

README.rst

@@ -97,13 +97,13 @@ Windows
 Install OpenSSL binaries from http://www.openssl.org/related/binaries.html and add
 it to Path:
 
-Swift client and Keystone client:
+Swift client and Keystone client::
 
     > pip install python-swiftclient
     > pip install python-keystoneclient
     > pip install freezer
 
-The basic Swift account configuration is needed to use freezer. Make sure python-swiftclient is installed.
+The basic Swift account configuration is needed to use freezer. Make sure python-swiftclient is installed::
 
     set OS_REGION_NAME=region-a.geo-1
     set OS_TENANT_ID=<account tenant>
@@ -135,7 +135,7 @@ The most simple backup execution is a direct file system backup::
 
     * On windows (need admin rights)*
     > freezerc --action backup --mode fs --backup-name testwindows
-    --path-to-backup "C:\path\to\backup" --volume C:\  --container freezer_windows
+    --path-to-backup "C:\path\to\backup" --container freezer_windows
     --log-file  C:\path\to\log\freezer.log
 
 By default --mode fs is set. The command would generate a compressed tar
@@ -646,4 +646,6 @@ Available options::
                             Set the SQL Server configuration file where freezer
                             retrieve the sql server instance. Following is an
                             example of config file: instance = <db-instance>
-      --volume VOLUME       Create a snapshot of the selected volume
+      --vssadmin VSSADMIN   Create a backup using a snapshot on windows
+                            using vssadmin. Options are: True and False,
+                            default is True

freezer/arguments.py

@@ -116,7 +116,10 @@ def backup_arguments(args_dict={}):
             'upload_limit': -1, 'always_level': False, 'version': False,
             'dry_run': False, 'lvm_snapsize': False,
             'restore_abs_path': False, 'log_file': None,
-            'upload': True, 'mode': 'fs', 'action': 'backup'}
+            'upload': True, 'mode': 'fs', 'action': 'backup',
+            'vssadmin': True, 'shadow': '', 'shadow_path': '',
+            'windows_volume': ''
+        }
 
     # Generate a new argparse istance and inherit options from config parse
     arg_parser = argparse.ArgumentParser(
@@ -376,9 +379,10 @@ def backup_arguments(args_dict={}):
         instance = <db-instance>''',
         dest='sql_server_conf', default=False)
     arg_parser.add_argument(
-        '--volume', action='store',
+        '--vssadmin', action='store',
-        help='Create a snapshot of the selected volume',
+        help='''Create a backup using a snapshot on windows
-        dest='volume', default=False)
+        using vssadmin. Options are: True and False, default is True''',
+        dest='vssadmin', default=True)
 
     arg_parser.set_defaults(**defaults)
     backup_args = arg_parser.parse_args()
@@ -386,7 +390,7 @@ def backup_arguments(args_dict={}):
     # windows bin
     path_to_binaries = os.path.dirname(os.path.abspath(__file__))
 
-    # Intercept command line arguments if you are not using the CLI
+    # Intercept command line arguments if you are not using the CLIvss
     if args_dict:
         backup_args.__dict__.update(args_dict)
 
@@ -463,6 +467,13 @@ def backup_arguments(args_dict={}):
     backup_args.__dict__['shadow'] = ''
     backup_args.__dict__['shadow_path'] = ''
     backup_args.__dict__['file_name'] = ''
+    if is_windows():
+        if backup_args.path_to_backup:
+            backup_args.__dict__['windows_volume'] = \
+                backup_args.path_to_backup[:3]
+
+        if backup_args.vssadmin == 'False' or backup_args.vssadmin == 'false':
+            backup_args.vssadmin = False
 
     backup_args.__dict__['meta_data'] = {}
     backup_args.__dict__['meta_data_file'] = ''

freezer/backup.py

@@ -33,8 +33,8 @@ from freezer.swift import add_object, manifest_upload
 from freezer.utils import gen_manifest_meta, add_host_name_ts_level
 from freezer.vss import vss_create_shadow_copy
 from freezer.vss import vss_delete_shadow_copy
-from freezer.vss import start_sql_server
+from freezer.winutils import start_sql_server
-from freezer.vss import stop_sql_server
+from freezer.winutils import stop_sql_server
 from freezer.winutils import use_shadow
 from freezer.winutils import is_windows
 from freezer import swift
@@ -63,6 +63,9 @@ def backup_mode_sql_server(backup_opt_dict, time_stamp, manifest_meta_dict):
         stop_sql_server(backup_opt_dict)
         backup_mode_fs(backup_opt_dict, time_stamp, manifest_meta_dict)
     finally:
+        if not backup_opt_dict.vssadmin:
+            # if vssadmin is false, wait until the backup is complete
+            # to start sql server again
             start_sql_server(backup_opt_dict)
 
 
@@ -237,10 +240,14 @@ def backup_mode_fs(backup_opt_dict, time_stamp, manifest_meta_dict):
     try:
 
         if is_windows():
-            # Create a shadow copy.
+            if backup_opt_dict.vssadmin:
                 # Create a shadow copy.
                 backup_opt_dict.shadow_path, backup_opt_dict.shadow = \
-                vss_create_shadow_copy(backup_opt_dict.volume)
+                    vss_create_shadow_copy(backup_opt_dict.windows_volume)
+
+                # execute this after the snapshot creation
+                if backup_opt_dict.mode == 'sqlserver':
+                    start_sql_server(backup_opt_dict)
 
         else:
             # If lvm_auto_snap is true, the volume group and volume name will
@@ -335,7 +342,7 @@ def backup_mode_fs(backup_opt_dict, time_stamp, manifest_meta_dict):
         if is_windows():
             # Delete the shadow copy after the backup
             vss_delete_shadow_copy(backup_opt_dict.shadow,
-                                   backup_opt_dict.volume)
+                                   backup_opt_dict.windows_volume)
         else:
             # Unmount and remove lvm snapshot volume
             lvm_snap_remove(backup_opt_dict)

freezer/swift.py

@@ -382,7 +382,7 @@ def add_object(
             package_name, time_stamp,
             backup_opt_dict.max_segment_size, file_chunk_index)
         add_chunk(backup_opt_dict.client_manager,
-                  backup_opt_dict.container_segment,
+                  backup_opt_dict.container_segments,
                   package_name, file_chunk)
 
 

freezer/vss.py

@@ -19,11 +19,11 @@ import logging
 import os
 
 
-def vss_create_shadow_copy(volume):
+def vss_create_shadow_copy(windows_volume):
     """
     Create a new shadow copy for the specified volume
 
-    Windows registry path for vss:
+    Windows registry path for vssadmin:
     HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\VSS\Settings
 
     MaxShadowCopies
@@ -64,10 +64,10 @@ def vss_create_shadow_copy(volume):
         (out, err) = create_subprocess(['powershell.exe',
                                         '-executionpolicy', 'unrestricted',
                                         '-command', script,
-                                        '-volume', volume])
+                                        '-volume', windows_volume])
         if err != '':
             raise Exception('[*] Error creating a new shadow copy on {0}'
-                            ', error {1}' .format(volume, err))
+                            ', error {1}' .format(windows_volume, err))
 
         for line in out.split('\n'):
             if 'symbolic' in line:
@@ -82,7 +82,7 @@ def vss_create_shadow_copy(volume):
         return shadow_path, shadow_id
 
 
-def vss_delete_shadow_copy(shadow_id, volume):
+def vss_delete_shadow_copy(shadow_id, windows_volume):
     """
     Delete a shadow copy from the volume with the given shadow_id
     :param shadow_id: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
@@ -98,38 +98,12 @@ def vss_delete_shadow_copy(shadow_id, volume):
                             ', error {1}' .format(shadow_id, err))
 
         try:
-            os.rmdir(os.path.join(volume, 'shadowcopy'))
+            os.rmdir(os.path.join(windows_volume, 'shadowcopy'))
         except Exception:
             logging.error('Failed to delete shadow copy symlink {0}'.
-                          format(os.path.join(volume, 'shadowcopy')))
+                          format(os.path.join(windows_volume, 'shadowcopy')))
 
         logging.info('[*] Deleting shadow copy {0}'.
                      format(shadow_id))
 
         return True
-
-
-def stop_sql_server(backup_opt_dict):
-    """ Stop a SQL Server instance to perform the backup of the db files """
-
-    logging.info('[*] Stopping SQL Server for backup')
-    with DisableFileSystemRedirection():
-        cmd = 'net stop "SQL Server ({0})"'\
-            .format(backup_opt_dict.sql_server_instance)
-        (out, err) = create_subprocess(cmd)
-        if err != '':
-            raise Exception('[*] Error while stopping SQL Server,'
-                            ', error {0}'.format(err))
-
-
-def start_sql_server(backup_opt_dict):
-    """ Start the SQL Server instance after the backup is completed """
-
-    with DisableFileSystemRedirection():
-        cmd = 'net start "SQL Server ({0})"'\
-            .format(backup_opt_dict.sql_server_instance)
-        (out, err) = create_subprocess(cmd)
-        if err != '':
-            raise Exception('[*] Error while starting SQL Server'
-                            ', error {0}'.format(err))
-        logging.info('[*] SQL Server back to normal')

freezer/winutils.py

@@ -17,8 +17,12 @@
 # Hudson (tjh@cryptsoft.com).
 # ========================================================================
 
-import sys
+
 import ctypes
+import logging
+import sys
+
+from freezer.utils import create_subprocess
 
 
 def is_windows():
@@ -51,9 +55,10 @@ class DisableFileSystemRedirection:
             self._revert(self.old_value)
 
 
-def use_shadow(to_backup, volume):
+def use_shadow(to_backup, windows_volume):
     """ add the shadow path to the backup directory """
-    return to_backup.replace(volume, '{0}shadowcopy\\'.format(volume))
+    return to_backup.replace(windows_volume, '{0}freezer_shadowcopy\\'
+                             .format(windows_volume))
 
 
 def clean_tar_command(tar_cmd):
@@ -69,3 +74,29 @@ def clean_tar_command(tar_cmd):
 def add_gzip_to_command(tar_cmd):
     gzip_cmd = 'gzip -7'
     return '{0} | {1}'.format(tar_cmd, gzip_cmd)
+
+
+def stop_sql_server(backup_opt_dict):
+    """ Stop a SQL Server instance to perform the backup of the db files """
+
+    logging.info('[*] Stopping SQL Server for backup')
+    with DisableFileSystemRedirection():
+        cmd = 'net stop "SQL Server ({0})"'\
+            .format(backup_opt_dict.sql_server_instance)
+        (out, err) = create_subprocess(cmd)
+        if err != '':
+            raise Exception('[*] Error while stopping SQL Server,'
+                            ', error {0}'.format(err))
+
+
+def start_sql_server(backup_opt_dict):
+    """ Start the SQL Server instance after the backup is completed """
+
+    with DisableFileSystemRedirection():
+        cmd = 'net start "SQL Server ({0})"'\
+            .format(backup_opt_dict.sql_server_instance)
+        (out, err) = create_subprocess(cmd)
+        if err != '':
+            raise Exception('[*] Error while starting SQL Server'
+                            ', error {0}'.format(err))
+        logging.info('[*] SQL Server back to normal')

tests/test_vss.py

@@ -27,76 +27,6 @@ import pytest
 
 class TestVss:
 
-    def test_start_sql_server(self, monkeypatch):
-        fake_disable_redirection = FakeDisableFileSystemRedirection()
-        backup_opt = BackupOpt1()
-        fakelogging = FakeLogging()
-        fakesubprocess = FakeSubProcess()
-        fakesubprocesspopen = fakesubprocess.Popen()
-
-        monkeypatch.setattr(
-            subprocess.Popen, 'communicate',
-            fakesubprocesspopen.communicate)
-        monkeypatch.setattr(
-            subprocess, 'Popen', fakesubprocesspopen)
-        monkeypatch.setattr(
-            winutils.DisableFileSystemRedirection, '__enter__',
-            fake_disable_redirection.__enter__)
-        monkeypatch.setattr(
-            winutils.DisableFileSystemRedirection, '__exit__',
-            fake_disable_redirection.__exit__)
-
-
-        monkeypatch.setattr(logging, 'info', fakelogging.info)
-
-        assert vss.start_sql_server(backup_opt) is not False
-
-        fakesubprocess = FakeSubProcess3()
-        fakesubprocesspopen = fakesubprocess.Popen()
-
-        monkeypatch.setattr(
-            subprocess.Popen, 'communicate',
-            fakesubprocesspopen.communicate)
-        monkeypatch.setattr(
-            subprocess, 'Popen', fakesubprocesspopen)
-
-        pytest.raises(Exception, vss.start_sql_server(backup_opt))
-
-    def test_stop_sql_server(self, monkeypatch):
-        fake_disable_redirection = FakeDisableFileSystemRedirection()
-        backup_opt = BackupOpt1()
-        fakelogging = FakeLogging()
-        fakesubprocess = FakeSubProcess()
-        fakesubprocesspopen = fakesubprocess.Popen()
-
-        monkeypatch.setattr(
-            subprocess.Popen, 'communicate',
-            fakesubprocesspopen.communicate)
-        monkeypatch.setattr(
-            subprocess, 'Popen', fakesubprocesspopen)
-        monkeypatch.setattr(
-            winutils.DisableFileSystemRedirection, '__enter__',
-            fake_disable_redirection.__enter__)
-        monkeypatch.setattr(
-            winutils.DisableFileSystemRedirection, '__exit__',
-            fake_disable_redirection.__exit__)
-
-
-        monkeypatch.setattr(logging, 'info', fakelogging.info)
-
-        assert vss.start_sql_server(backup_opt) is not False
-
-        fakesubprocess = FakeSubProcess3()
-        fakesubprocesspopen = fakesubprocess.Popen()
-
-        monkeypatch.setattr(
-            subprocess.Popen, 'communicate',
-            fakesubprocesspopen.communicate)
-        monkeypatch.setattr(
-            subprocess, 'Popen', fakesubprocesspopen)
-
-        pytest.raises(Exception, vss.stop_sql_server(backup_opt))
-
     def test_vss_create_shadow_copy(self, monkeypatch):
         fake_disable_redirection = FakeDisableFileSystemRedirection()
         fakelogging = FakeLogging()

tests/test_winutils.py

@@ -17,7 +17,9 @@ from freezer.winutils import use_shadow
 from freezer.winutils import clean_tar_command
 from freezer.winutils import add_gzip_to_command
 from freezer.winutils import DisableFileSystemRedirection
+from freezer import winutils
 from commons import *
+import logging
 
 import pytest
 
@@ -33,7 +35,7 @@ class TestWinutils:
         test_volume = 'C:'
         test_volume2 = 'C:\\'
         path = 'C:\\Users\\Test'
-        expected = 'C:\\shadowcopy\\Users\\Test'
+        expected = 'C:\\freezer_shadowcopy\\Users\\Test'
         assert use_shadow(path, test_volume2) == expected
 
         # test if the volume format is incorrect
@@ -68,3 +70,70 @@ class TestWinutils:
         pytest.raises(Exception, fake_disable_redirection.__enter__)
         pytest.raises(Exception, fake_disable_redirection.__exit__)
 
+    def test_start_sql_server(self, monkeypatch):
+        fake_disable_redirection = FakeDisableFileSystemRedirection()
+        backup_opt = BackupOpt1()
+        fakelogging = FakeLogging()
+        fakesubprocess = FakeSubProcess()
+        fakesubprocesspopen = fakesubprocess.Popen()
+
+        monkeypatch.setattr(
+            subprocess.Popen, 'communicate',
+            fakesubprocesspopen.communicate)
+        monkeypatch.setattr(
+            subprocess, 'Popen', fakesubprocesspopen)
+        monkeypatch.setattr(
+            winutils.DisableFileSystemRedirection, '__enter__',
+            fake_disable_redirection.__enter__)
+        monkeypatch.setattr(
+            winutils.DisableFileSystemRedirection, '__exit__',
+            fake_disable_redirection.__exit__)
+
+        monkeypatch.setattr(logging, 'info', fakelogging.info)
+
+        assert winutils.start_sql_server(backup_opt) is not False
+
+        fakesubprocess = FakeSubProcess3()
+        fakesubprocesspopen = fakesubprocess.Popen()
+
+        monkeypatch.setattr(
+            subprocess.Popen, 'communicate',
+            fakesubprocesspopen.communicate)
+        monkeypatch.setattr(
+            subprocess, 'Popen', fakesubprocesspopen)
+
+        pytest.raises(Exception, winutils.start_sql_server(backup_opt))
+
+    def test_stop_sql_server(self, monkeypatch):
+        fake_disable_redirection = FakeDisableFileSystemRedirection()
+        backup_opt = BackupOpt1()
+        fakelogging = FakeLogging()
+        fakesubprocess = FakeSubProcess()
+        fakesubprocesspopen = fakesubprocess.Popen()
+
+        monkeypatch.setattr(
+            subprocess.Popen, 'communicate',
+            fakesubprocesspopen.communicate)
+        monkeypatch.setattr(
+            subprocess, 'Popen', fakesubprocesspopen)
+        monkeypatch.setattr(
+            winutils.DisableFileSystemRedirection, '__enter__',
+            fake_disable_redirection.__enter__)
+        monkeypatch.setattr(
+            winutils.DisableFileSystemRedirection, '__exit__',
+            fake_disable_redirection.__exit__)
+
+        monkeypatch.setattr(logging, 'info', fakelogging.info)
+
+        assert winutils.start_sql_server(backup_opt) is not False
+
+        fakesubprocess = FakeSubProcess3()
+        fakesubprocesspopen = fakesubprocess.Popen()
+
+        monkeypatch.setattr(
+            subprocess.Popen, 'communicate',
+            fakesubprocesspopen.communicate)
+        monkeypatch.setattr(
+            subprocess, 'Popen', fakesubprocesspopen)
+
+        pytest.raises(Exception, winutils.stop_sql_server(backup_opt))