From 257ea34d238d5b3156eb253f081cac27c128465e Mon Sep 17 00:00:00 2001 From: Memo Garcia Date: Tue, 9 Jun 2015 10:42:06 +0100 Subject: [PATCH] Reduce SQL Server downtime and Snapshot option on windows CLI argument for snapshots on windows (vssadmin) Reduce downtime with SQL Server if a snapshot is available Change-Id: Iaae4d028cd19e1732925ec5bd86665b839528cd3 --- README.rst | 10 +++--- freezer/arguments.py | 21 +++++++++--- freezer/backup.py | 23 ++++++++----- freezer/swift.py | 2 +- freezer/vss.py | 40 ++++------------------- freezer/winutils.py | 37 +++++++++++++++++++-- tests/test_arguments.py | 2 +- tests/test_vss.py | 72 +---------------------------------------- tests/test_winutils.py | 71 +++++++++++++++++++++++++++++++++++++++- 9 files changed, 151 insertions(+), 127 deletions(-) diff --git a/README.rst b/README.rst index f7396247..a0f7533c 100644 --- a/README.rst +++ b/README.rst @@ -97,13 +97,13 @@ Windows Install OpenSSL binaries from http://www.openssl.org/related/binaries.html and add it to Path: -Swift client and Keystone client: +Swift client and Keystone client:: > pip install python-swiftclient > pip install python-keystoneclient > pip install freezer -The basic Swift account configuration is needed to use freezer. Make sure python-swiftclient is installed. +The basic Swift account configuration is needed to use freezer. Make sure python-swiftclient is installed:: set OS_REGION_NAME=region-a.geo-1 set OS_TENANT_ID= @@ -135,7 +135,7 @@ The most simple backup execution is a direct file system backup:: * On windows (need admin rights)* > freezerc --action backup --mode fs --backup-name testwindows - --path-to-backup "C:\path\to\backup" --volume C:\ --container freezer_windows + --path-to-backup "C:\path\to\backup" --container freezer_windows --log-file C:\path\to\log\freezer.log By default --mode fs is set. The command would generate a compressed tar @@ -646,4 +646,6 @@ Available options:: Set the SQL Server configuration file where freezer retrieve the sql server instance. Following is an example of config file: instance = - --volume VOLUME Create a snapshot of the selected volume \ No newline at end of file + --vssadmin VSSADMIN Create a backup using a snapshot on windows + using vssadmin. Options are: True and False, + default is True diff --git a/freezer/arguments.py b/freezer/arguments.py index ca44f932..7d2149f4 100644 --- a/freezer/arguments.py +++ b/freezer/arguments.py @@ -116,7 +116,10 @@ def backup_arguments(args_dict={}): 'upload_limit': -1, 'always_level': False, 'version': False, 'dry_run': False, 'lvm_snapsize': False, 'restore_abs_path': False, 'log_file': None, - 'upload': True, 'mode': 'fs', 'action': 'backup'} + 'upload': True, 'mode': 'fs', 'action': 'backup', + 'vssadmin': True, 'shadow': '', 'shadow_path': '', + 'windows_volume': '' + } # Generate a new argparse istance and inherit options from config parse arg_parser = argparse.ArgumentParser( @@ -376,9 +379,10 @@ def backup_arguments(args_dict={}): instance = ''', dest='sql_server_conf', default=False) arg_parser.add_argument( - '--volume', action='store', - help='Create a snapshot of the selected volume', - dest='volume', default=False) + '--vssadmin', action='store', + help='''Create a backup using a snapshot on windows + using vssadmin. Options are: True and False, default is True''', + dest='vssadmin', default=True) arg_parser.set_defaults(**defaults) backup_args = arg_parser.parse_args() @@ -386,7 +390,7 @@ def backup_arguments(args_dict={}): # windows bin path_to_binaries = os.path.dirname(os.path.abspath(__file__)) - # Intercept command line arguments if you are not using the CLI + # Intercept command line arguments if you are not using the CLIvss if args_dict: backup_args.__dict__.update(args_dict) @@ -463,6 +467,13 @@ def backup_arguments(args_dict={}): backup_args.__dict__['shadow'] = '' backup_args.__dict__['shadow_path'] = '' backup_args.__dict__['file_name'] = '' + if is_windows(): + if backup_args.path_to_backup: + backup_args.__dict__['windows_volume'] = \ + backup_args.path_to_backup[:3] + + if backup_args.vssadmin == 'False' or backup_args.vssadmin == 'false': + backup_args.vssadmin = False backup_args.__dict__['meta_data'] = {} backup_args.__dict__['meta_data_file'] = '' diff --git a/freezer/backup.py b/freezer/backup.py index 6588b834..132173f3 100644 --- a/freezer/backup.py +++ b/freezer/backup.py @@ -33,8 +33,8 @@ from freezer.swift import add_object, manifest_upload from freezer.utils import gen_manifest_meta, add_host_name_ts_level from freezer.vss import vss_create_shadow_copy from freezer.vss import vss_delete_shadow_copy -from freezer.vss import start_sql_server -from freezer.vss import stop_sql_server +from freezer.winutils import start_sql_server +from freezer.winutils import stop_sql_server from freezer.winutils import use_shadow from freezer.winutils import is_windows from freezer import swift @@ -63,7 +63,10 @@ def backup_mode_sql_server(backup_opt_dict, time_stamp, manifest_meta_dict): stop_sql_server(backup_opt_dict) backup_mode_fs(backup_opt_dict, time_stamp, manifest_meta_dict) finally: - start_sql_server(backup_opt_dict) + if not backup_opt_dict.vssadmin: + # if vssadmin is false, wait until the backup is complete + # to start sql server again + start_sql_server(backup_opt_dict) def backup_mode_mysql(backup_opt_dict, time_stamp, manifest_meta_dict): @@ -237,10 +240,14 @@ def backup_mode_fs(backup_opt_dict, time_stamp, manifest_meta_dict): try: if is_windows(): - # Create a shadow copy. - # Create a shadow copy. - backup_opt_dict.shadow_path, backup_opt_dict.shadow = \ - vss_create_shadow_copy(backup_opt_dict.volume) + if backup_opt_dict.vssadmin: + # Create a shadow copy. + backup_opt_dict.shadow_path, backup_opt_dict.shadow = \ + vss_create_shadow_copy(backup_opt_dict.windows_volume) + + # execute this after the snapshot creation + if backup_opt_dict.mode == 'sqlserver': + start_sql_server(backup_opt_dict) else: # If lvm_auto_snap is true, the volume group and volume name will @@ -335,7 +342,7 @@ def backup_mode_fs(backup_opt_dict, time_stamp, manifest_meta_dict): if is_windows(): # Delete the shadow copy after the backup vss_delete_shadow_copy(backup_opt_dict.shadow, - backup_opt_dict.volume) + backup_opt_dict.windows_volume) else: # Unmount and remove lvm snapshot volume lvm_snap_remove(backup_opt_dict) diff --git a/freezer/swift.py b/freezer/swift.py index fee9d7f2..c385006e 100644 --- a/freezer/swift.py +++ b/freezer/swift.py @@ -382,7 +382,7 @@ def add_object( package_name, time_stamp, backup_opt_dict.max_segment_size, file_chunk_index) add_chunk(backup_opt_dict.client_manager, - backup_opt_dict.container_segment, + backup_opt_dict.container_segments, package_name, file_chunk) diff --git a/freezer/vss.py b/freezer/vss.py index babeb1cd..4731c0a5 100644 --- a/freezer/vss.py +++ b/freezer/vss.py @@ -19,11 +19,11 @@ import logging import os -def vss_create_shadow_copy(volume): +def vss_create_shadow_copy(windows_volume): """ Create a new shadow copy for the specified volume - Windows registry path for vss: + Windows registry path for vssadmin: HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\VSS\Settings MaxShadowCopies @@ -64,10 +64,10 @@ def vss_create_shadow_copy(volume): (out, err) = create_subprocess(['powershell.exe', '-executionpolicy', 'unrestricted', '-command', script, - '-volume', volume]) + '-volume', windows_volume]) if err != '': raise Exception('[*] Error creating a new shadow copy on {0}' - ', error {1}' .format(volume, err)) + ', error {1}' .format(windows_volume, err)) for line in out.split('\n'): if 'symbolic' in line: @@ -82,7 +82,7 @@ def vss_create_shadow_copy(volume): return shadow_path, shadow_id -def vss_delete_shadow_copy(shadow_id, volume): +def vss_delete_shadow_copy(shadow_id, windows_volume): """ Delete a shadow copy from the volume with the given shadow_id :param shadow_id: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX @@ -98,38 +98,12 @@ def vss_delete_shadow_copy(shadow_id, volume): ', error {1}' .format(shadow_id, err)) try: - os.rmdir(os.path.join(volume, 'shadowcopy')) + os.rmdir(os.path.join(windows_volume, 'shadowcopy')) except Exception: logging.error('Failed to delete shadow copy symlink {0}'. - format(os.path.join(volume, 'shadowcopy'))) + format(os.path.join(windows_volume, 'shadowcopy'))) logging.info('[*] Deleting shadow copy {0}'. format(shadow_id)) return True - - -def stop_sql_server(backup_opt_dict): - """ Stop a SQL Server instance to perform the backup of the db files """ - - logging.info('[*] Stopping SQL Server for backup') - with DisableFileSystemRedirection(): - cmd = 'net stop "SQL Server ({0})"'\ - .format(backup_opt_dict.sql_server_instance) - (out, err) = create_subprocess(cmd) - if err != '': - raise Exception('[*] Error while stopping SQL Server,' - ', error {0}'.format(err)) - - -def start_sql_server(backup_opt_dict): - """ Start the SQL Server instance after the backup is completed """ - - with DisableFileSystemRedirection(): - cmd = 'net start "SQL Server ({0})"'\ - .format(backup_opt_dict.sql_server_instance) - (out, err) = create_subprocess(cmd) - if err != '': - raise Exception('[*] Error while starting SQL Server' - ', error {0}'.format(err)) - logging.info('[*] SQL Server back to normal') diff --git a/freezer/winutils.py b/freezer/winutils.py index 591409e5..fb41b40f 100644 --- a/freezer/winutils.py +++ b/freezer/winutils.py @@ -17,8 +17,12 @@ # Hudson (tjh@cryptsoft.com). # ======================================================================== -import sys + import ctypes +import logging +import sys + +from freezer.utils import create_subprocess def is_windows(): @@ -51,9 +55,10 @@ class DisableFileSystemRedirection: self._revert(self.old_value) -def use_shadow(to_backup, volume): +def use_shadow(to_backup, windows_volume): """ add the shadow path to the backup directory """ - return to_backup.replace(volume, '{0}shadowcopy\\'.format(volume)) + return to_backup.replace(windows_volume, '{0}freezer_shadowcopy\\' + .format(windows_volume)) def clean_tar_command(tar_cmd): @@ -69,3 +74,29 @@ def clean_tar_command(tar_cmd): def add_gzip_to_command(tar_cmd): gzip_cmd = 'gzip -7' return '{0} | {1}'.format(tar_cmd, gzip_cmd) + + +def stop_sql_server(backup_opt_dict): + """ Stop a SQL Server instance to perform the backup of the db files """ + + logging.info('[*] Stopping SQL Server for backup') + with DisableFileSystemRedirection(): + cmd = 'net stop "SQL Server ({0})"'\ + .format(backup_opt_dict.sql_server_instance) + (out, err) = create_subprocess(cmd) + if err != '': + raise Exception('[*] Error while stopping SQL Server,' + ', error {0}'.format(err)) + + +def start_sql_server(backup_opt_dict): + """ Start the SQL Server instance after the backup is completed """ + + with DisableFileSystemRedirection(): + cmd = 'net start "SQL Server ({0})"'\ + .format(backup_opt_dict.sql_server_instance) + (out, err) = create_subprocess(cmd) + if err != '': + raise Exception('[*] Error while starting SQL Server' + ', error {0}'.format(err)) + logging.info('[*] SQL Server back to normal') diff --git a/tests/test_arguments.py b/tests/test_arguments.py index 5e792acc..a4e5303b 100644 --- a/tests/test_arguments.py +++ b/tests/test_arguments.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from freezer.arguments import backup_arguments, alter_proxy +from freezer.arguments import backup_arguments, alter_proxy import argparse from commons import * import sys diff --git a/tests/test_vss.py b/tests/test_vss.py index 8f9c28b1..e804ea24 100644 --- a/tests/test_vss.py +++ b/tests/test_vss.py @@ -27,76 +27,6 @@ import pytest class TestVss: - def test_start_sql_server(self, monkeypatch): - fake_disable_redirection = FakeDisableFileSystemRedirection() - backup_opt = BackupOpt1() - fakelogging = FakeLogging() - fakesubprocess = FakeSubProcess() - fakesubprocesspopen = fakesubprocess.Popen() - - monkeypatch.setattr( - subprocess.Popen, 'communicate', - fakesubprocesspopen.communicate) - monkeypatch.setattr( - subprocess, 'Popen', fakesubprocesspopen) - monkeypatch.setattr( - winutils.DisableFileSystemRedirection, '__enter__', - fake_disable_redirection.__enter__) - monkeypatch.setattr( - winutils.DisableFileSystemRedirection, '__exit__', - fake_disable_redirection.__exit__) - - - monkeypatch.setattr(logging, 'info', fakelogging.info) - - assert vss.start_sql_server(backup_opt) is not False - - fakesubprocess = FakeSubProcess3() - fakesubprocesspopen = fakesubprocess.Popen() - - monkeypatch.setattr( - subprocess.Popen, 'communicate', - fakesubprocesspopen.communicate) - monkeypatch.setattr( - subprocess, 'Popen', fakesubprocesspopen) - - pytest.raises(Exception, vss.start_sql_server(backup_opt)) - - def test_stop_sql_server(self, monkeypatch): - fake_disable_redirection = FakeDisableFileSystemRedirection() - backup_opt = BackupOpt1() - fakelogging = FakeLogging() - fakesubprocess = FakeSubProcess() - fakesubprocesspopen = fakesubprocess.Popen() - - monkeypatch.setattr( - subprocess.Popen, 'communicate', - fakesubprocesspopen.communicate) - monkeypatch.setattr( - subprocess, 'Popen', fakesubprocesspopen) - monkeypatch.setattr( - winutils.DisableFileSystemRedirection, '__enter__', - fake_disable_redirection.__enter__) - monkeypatch.setattr( - winutils.DisableFileSystemRedirection, '__exit__', - fake_disable_redirection.__exit__) - - - monkeypatch.setattr(logging, 'info', fakelogging.info) - - assert vss.start_sql_server(backup_opt) is not False - - fakesubprocess = FakeSubProcess3() - fakesubprocesspopen = fakesubprocess.Popen() - - monkeypatch.setattr( - subprocess.Popen, 'communicate', - fakesubprocesspopen.communicate) - monkeypatch.setattr( - subprocess, 'Popen', fakesubprocesspopen) - - pytest.raises(Exception, vss.stop_sql_server(backup_opt)) - def test_vss_create_shadow_copy(self, monkeypatch): fake_disable_redirection = FakeDisableFileSystemRedirection() fakelogging = FakeLogging() @@ -173,4 +103,4 @@ class TestVss: monkeypatch.setattr( subprocess, 'Popen', fakesubprocesspopen) - assert vss.vss_delete_shadow_copy('shadow_id', 'C:\\') is True \ No newline at end of file + assert vss.vss_delete_shadow_copy('shadow_id', 'C:\\') is True diff --git a/tests/test_winutils.py b/tests/test_winutils.py index 662598b1..9ec91b35 100644 --- a/tests/test_winutils.py +++ b/tests/test_winutils.py @@ -17,7 +17,9 @@ from freezer.winutils import use_shadow from freezer.winutils import clean_tar_command from freezer.winutils import add_gzip_to_command from freezer.winutils import DisableFileSystemRedirection +from freezer import winutils from commons import * +import logging import pytest @@ -33,7 +35,7 @@ class TestWinutils: test_volume = 'C:' test_volume2 = 'C:\\' path = 'C:\\Users\\Test' - expected = 'C:\\shadowcopy\\Users\\Test' + expected = 'C:\\freezer_shadowcopy\\Users\\Test' assert use_shadow(path, test_volume2) == expected # test if the volume format is incorrect @@ -68,3 +70,70 @@ class TestWinutils: pytest.raises(Exception, fake_disable_redirection.__enter__) pytest.raises(Exception, fake_disable_redirection.__exit__) + def test_start_sql_server(self, monkeypatch): + fake_disable_redirection = FakeDisableFileSystemRedirection() + backup_opt = BackupOpt1() + fakelogging = FakeLogging() + fakesubprocess = FakeSubProcess() + fakesubprocesspopen = fakesubprocess.Popen() + + monkeypatch.setattr( + subprocess.Popen, 'communicate', + fakesubprocesspopen.communicate) + monkeypatch.setattr( + subprocess, 'Popen', fakesubprocesspopen) + monkeypatch.setattr( + winutils.DisableFileSystemRedirection, '__enter__', + fake_disable_redirection.__enter__) + monkeypatch.setattr( + winutils.DisableFileSystemRedirection, '__exit__', + fake_disable_redirection.__exit__) + + monkeypatch.setattr(logging, 'info', fakelogging.info) + + assert winutils.start_sql_server(backup_opt) is not False + + fakesubprocess = FakeSubProcess3() + fakesubprocesspopen = fakesubprocess.Popen() + + monkeypatch.setattr( + subprocess.Popen, 'communicate', + fakesubprocesspopen.communicate) + monkeypatch.setattr( + subprocess, 'Popen', fakesubprocesspopen) + + pytest.raises(Exception, winutils.start_sql_server(backup_opt)) + + def test_stop_sql_server(self, monkeypatch): + fake_disable_redirection = FakeDisableFileSystemRedirection() + backup_opt = BackupOpt1() + fakelogging = FakeLogging() + fakesubprocess = FakeSubProcess() + fakesubprocesspopen = fakesubprocess.Popen() + + monkeypatch.setattr( + subprocess.Popen, 'communicate', + fakesubprocesspopen.communicate) + monkeypatch.setattr( + subprocess, 'Popen', fakesubprocesspopen) + monkeypatch.setattr( + winutils.DisableFileSystemRedirection, '__enter__', + fake_disable_redirection.__enter__) + monkeypatch.setattr( + winutils.DisableFileSystemRedirection, '__exit__', + fake_disable_redirection.__exit__) + + monkeypatch.setattr(logging, 'info', fakelogging.info) + + assert winutils.start_sql_server(backup_opt) is not False + + fakesubprocess = FakeSubProcess3() + fakesubprocesspopen = fakesubprocess.Popen() + + monkeypatch.setattr( + subprocess.Popen, 'communicate', + fakesubprocesspopen.communicate) + monkeypatch.setattr( + subprocess, 'Popen', fakesubprocesspopen) + + pytest.raises(Exception, winutils.stop_sql_server(backup_opt))