openstack/freezer
Code Issues Proposed changes

Reduce SQL Server downtime and Snapshot option on windows

Browse Source 
CLI argument for snapshots on windows (vssadmin)
Reduce downtime with SQL Server if a snapshot is available

Change-Id: Iaae4d028cd19e1732925ec5bd86665b839528cd3
This commit is contained in:
Memo Garcia 2015-06-09 10:42:06 +01:00
parent 4a863ca7f3
commit 257ea34d23
9 changed files with 151 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
README.rst
View File

@ -97,13 +97,13 @@ Windows
Install OpenSSL binaries from http://www.openssl.org/related/binaries.html and add
it to Path:
Swift client and Keystone client:
Swift client and Keystone client::
> pip install python-swiftclient
> pip install python-keystoneclient
> pip install freezer
The basic Swift account configuration is needed to use freezer. Make sure python-swiftclient is installed.
The basic Swift account configuration is needed to use freezer. Make sure python-swiftclient is installed::
set OS_REGION_NAME=region-a.geo-1
set OS_TENANT_ID=<account tenant>
@ -135,7 +135,7 @@ The most simple backup execution is a direct file system backup::
* On windows (need admin rights)*
> freezerc --action backup --mode fs --backup-name testwindows
--path-to-backup "C:\path\to\backup" --volume C:\ --container freezer_windows
--path-to-backup "C:\path\to\backup" --container freezer_windows
--log-file C:\path\to\log\freezer.log
By default --mode fs is set. The command would generate a compressed tar
@ -646,4 +646,6 @@ Available options::
Set the SQL Server configuration file where freezer
retrieve the sql server instance. Following is an
example of config file: instance = <db-instance>
--volume VOLUME Create a snapshot of the selected volume
--vssadmin VSSADMIN Create a backup using a snapshot on windows
using vssadmin. Options are: True and False,
default is True

21
freezer/arguments.py
View File

@ -116,7 +116,10 @@ def backup_arguments(args_dict={}):
'upload_limit': -1, 'always_level': False, 'version': False,
'dry_run': False, 'lvm_snapsize': False,
'restore_abs_path': False, 'log_file': None,
'upload': True, 'mode': 'fs', 'action': 'backup'}
'upload': True, 'mode': 'fs', 'action': 'backup',
'vssadmin': True, 'shadow': '', 'shadow_path': '',
'windows_volume': ''
}
# Generate a new argparse istance and inherit options from config parse
arg_parser = argparse.ArgumentParser(
@ -376,9 +379,10 @@ def backup_arguments(args_dict={}):
instance = <db-instance>''',
dest='sql_server_conf', default=False)
arg_parser.add_argument(
'--volume', action='store',
help='Create a snapshot of the selected volume',
dest='volume', default=False)
'--vssadmin', action='store',
help='''Create a backup using a snapshot on windows
using vssadmin. Options are: True and False, default is True''',
dest='vssadmin', default=True)
arg_parser.set_defaults(**defaults)
backup_args = arg_parser.parse_args()
@ -386,7 +390,7 @@ def backup_arguments(args_dict={}):
# windows bin
path_to_binaries = os.path.dirname(os.path.abspath(__file__))
# Intercept command line arguments if you are not using the CLI
# Intercept command line arguments if you are not using the CLIvss
if args_dict:
backup_args.__dict__.update(args_dict)
@ -463,6 +467,13 @@ def backup_arguments(args_dict={}):
backup_args.__dict__['shadow'] = ''
backup_args.__dict__['shadow_path'] = ''
backup_args.__dict__['file_name'] = ''
if is_windows():
if backup_args.path_to_backup:
backup_args.__dict__['windows_volume'] = \
backup_args.path_to_backup[:3]
if backup_args.vssadmin == 'False' or backup_args.vssadmin == 'false':
backup_args.vssadmin = False
backup_args.__dict__['meta_data'] = {}
backup_args.__dict__['meta_data_file'] = ''

23
freezer/backup.py
View File

@ -33,8 +33,8 @@ from freezer.swift import add_object, manifest_upload
from freezer.utils import gen_manifest_meta, add_host_name_ts_level
from freezer.vss import vss_create_shadow_copy
from freezer.vss import vss_delete_shadow_copy
from freezer.vss import start_sql_server
from freezer.vss import stop_sql_server
from freezer.winutils import start_sql_server
from freezer.winutils import stop_sql_server
from freezer.winutils import use_shadow
from freezer.winutils import is_windows
from freezer import swift
@ -63,7 +63,10 @@ def backup_mode_sql_server(backup_opt_dict, time_stamp, manifest_meta_dict):
stop_sql_server(backup_opt_dict)
backup_mode_fs(backup_opt_dict, time_stamp, manifest_meta_dict)
finally:
start_sql_server(backup_opt_dict)
if not backup_opt_dict.vssadmin:
# if vssadmin is false, wait until the backup is complete
# to start sql server again
start_sql_server(backup_opt_dict)
def backup_mode_mysql(backup_opt_dict, time_stamp, manifest_meta_dict):
@ -237,10 +240,14 @@ def backup_mode_fs(backup_opt_dict, time_stamp, manifest_meta_dict):
try:
if is_windows():
# Create a shadow copy.
# Create a shadow copy.
backup_opt_dict.shadow_path, backup_opt_dict.shadow = \
vss_create_shadow_copy(backup_opt_dict.volume)
if backup_opt_dict.vssadmin:
# Create a shadow copy.
backup_opt_dict.shadow_path, backup_opt_dict.shadow = \
vss_create_shadow_copy(backup_opt_dict.windows_volume)
# execute this after the snapshot creation
if backup_opt_dict.mode == 'sqlserver':
start_sql_server(backup_opt_dict)
else:
# If lvm_auto_snap is true, the volume group and volume name will
@ -335,7 +342,7 @@ def backup_mode_fs(backup_opt_dict, time_stamp, manifest_meta_dict):
if is_windows():
# Delete the shadow copy after the backup
vss_delete_shadow_copy(backup_opt_dict.shadow,
backup_opt_dict.volume)
backup_opt_dict.windows_volume)
else:
# Unmount and remove lvm snapshot volume
lvm_snap_remove(backup_opt_dict)

2
freezer/swift.py
View File

@ -382,7 +382,7 @@ def add_object(
package_name, time_stamp,
backup_opt_dict.max_segment_size, file_chunk_index)
add_chunk(backup_opt_dict.client_manager,
backup_opt_dict.container_segment,
backup_opt_dict.container_segments,
package_name, file_chunk)

40
freezer/vss.py
View File

@ -19,11 +19,11 @@ import logging
import os
def vss_create_shadow_copy(volume):
def vss_create_shadow_copy(windows_volume):
"""
Create a new shadow copy for the specified volume
Windows registry path for vss:
Windows registry path for vssadmin:
HKEY_LOCAL_MACHINE\System\CurrentControlSet\Services\VSS\Settings
MaxShadowCopies
@ -64,10 +64,10 @@ def vss_create_shadow_copy(volume):
(out, err) = create_subprocess(['powershell.exe',
'-executionpolicy', 'unrestricted',
'-command', script,
'-volume', volume])
'-volume', windows_volume])
if err != '':
raise Exception('[*] Error creating a new shadow copy on {0}'
', error {1}' .format(volume, err))
', error {1}' .format(windows_volume, err))
for line in out.split('\n'):
if 'symbolic' in line:
@ -82,7 +82,7 @@ def vss_create_shadow_copy(volume):
return shadow_path, shadow_id
def vss_delete_shadow_copy(shadow_id, volume):
def vss_delete_shadow_copy(shadow_id, windows_volume):
"""
Delete a shadow copy from the volume with the given shadow_id
:param shadow_id: XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX
@ -98,38 +98,12 @@ def vss_delete_shadow_copy(shadow_id, volume):
', error {1}' .format(shadow_id, err))
try:
os.rmdir(os.path.join(volume, 'shadowcopy'))
os.rmdir(os.path.join(windows_volume, 'shadowcopy'))
except Exception:
logging.error('Failed to delete shadow copy symlink {0}'.
format(os.path.join(volume, 'shadowcopy')))
format(os.path.join(windows_volume, 'shadowcopy')))
logging.info('[*] Deleting shadow copy {0}'.
format(shadow_id))
return True
def stop_sql_server(backup_opt_dict):
""" Stop a SQL Server instance to perform the backup of the db files """
logging.info('[*] Stopping SQL Server for backup')
with DisableFileSystemRedirection():
cmd = 'net stop "SQL Server ({0})"'\
.format(backup_opt_dict.sql_server_instance)
(out, err) = create_subprocess(cmd)
if err != '':
raise Exception('[*] Error while stopping SQL Server,'
', error {0}'.format(err))
def start_sql_server(backup_opt_dict):
""" Start the SQL Server instance after the backup is completed """
with DisableFileSystemRedirection():
cmd = 'net start "SQL Server ({0})"'\
.format(backup_opt_dict.sql_server_instance)
(out, err) = create_subprocess(cmd)
if err != '':
raise Exception('[*] Error while starting SQL Server'
', error {0}'.format(err))
logging.info('[*] SQL Server back to normal')

37
freezer/winutils.py
View File

@ -17,8 +17,12 @@
# Hudson (tjh@cryptsoft.com).
# ========================================================================
import sys
import ctypes
import logging
import sys
from freezer.utils import create_subprocess
def is_windows():
@ -51,9 +55,10 @@ class DisableFileSystemRedirection:
self._revert(self.old_value)
def use_shadow(to_backup, volume):
def use_shadow(to_backup, windows_volume):
""" add the shadow path to the backup directory """
return to_backup.replace(volume, '{0}shadowcopy\\'.format(volume))
return to_backup.replace(windows_volume, '{0}freezer_shadowcopy\\'
.format(windows_volume))
def clean_tar_command(tar_cmd):
@ -69,3 +74,29 @@ def clean_tar_command(tar_cmd):
def add_gzip_to_command(tar_cmd):
gzip_cmd = 'gzip -7'
return '{0} | {1}'.format(tar_cmd, gzip_cmd)
def stop_sql_server(backup_opt_dict):
""" Stop a SQL Server instance to perform the backup of the db files """
logging.info('[*] Stopping SQL Server for backup')
with DisableFileSystemRedirection():
cmd = 'net stop "SQL Server ({0})"'\
.format(backup_opt_dict.sql_server_instance)
(out, err) = create_subprocess(cmd)
if err != '':
raise Exception('[*] Error while stopping SQL Server,'
', error {0}'.format(err))
def start_sql_server(backup_opt_dict):
""" Start the SQL Server instance after the backup is completed """
with DisableFileSystemRedirection():
cmd = 'net start "SQL Server ({0})"'\
.format(backup_opt_dict.sql_server_instance)
(out, err) = create_subprocess(cmd)
if err != '':
raise Exception('[*] Error while starting SQL Server'
', error {0}'.format(err))
logging.info('[*] SQL Server back to normal')

2
tests/test_arguments.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python
from freezer.arguments import backup_arguments, alter_proxy
from freezer.arguments import backup_arguments, alter_proxy
import argparse
from commons import *
import sys

72
tests/test_vss.py
View File

@ -27,76 +27,6 @@ import pytest
class TestVss:
def test_start_sql_server(self, monkeypatch):
fake_disable_redirection = FakeDisableFileSystemRedirection()
backup_opt = BackupOpt1()
fakelogging = FakeLogging()
fakesubprocess = FakeSubProcess()
fakesubprocesspopen = fakesubprocess.Popen()
monkeypatch.setattr(
subprocess.Popen, 'communicate',
fakesubprocesspopen.communicate)
monkeypatch.setattr(
subprocess, 'Popen', fakesubprocesspopen)
monkeypatch.setattr(
winutils.DisableFileSystemRedirection, '__enter__',
fake_disable_redirection.__enter__)
monkeypatch.setattr(
winutils.DisableFileSystemRedirection, '__exit__',
fake_disable_redirection.__exit__)
monkeypatch.setattr(logging, 'info', fakelogging.info)
assert vss.start_sql_server(backup_opt) is not False
fakesubprocess = FakeSubProcess3()
fakesubprocesspopen = fakesubprocess.Popen()
monkeypatch.setattr(
subprocess.Popen, 'communicate',
fakesubprocesspopen.communicate)
monkeypatch.setattr(
subprocess, 'Popen', fakesubprocesspopen)
pytest.raises(Exception, vss.start_sql_server(backup_opt))
def test_stop_sql_server(self, monkeypatch):
fake_disable_redirection = FakeDisableFileSystemRedirection()
backup_opt = BackupOpt1()
fakelogging = FakeLogging()
fakesubprocess = FakeSubProcess()
fakesubprocesspopen = fakesubprocess.Popen()
monkeypatch.setattr(
subprocess.Popen, 'communicate',
fakesubprocesspopen.communicate)
monkeypatch.setattr(
subprocess, 'Popen', fakesubprocesspopen)
monkeypatch.setattr(
winutils.DisableFileSystemRedirection, '__enter__',
fake_disable_redirection.__enter__)
monkeypatch.setattr(
winutils.DisableFileSystemRedirection, '__exit__',
fake_disable_redirection.__exit__)
monkeypatch.setattr(logging, 'info', fakelogging.info)
assert vss.start_sql_server(backup_opt) is not False
fakesubprocess = FakeSubProcess3()
fakesubprocesspopen = fakesubprocess.Popen()
monkeypatch.setattr(
subprocess.Popen, 'communicate',
fakesubprocesspopen.communicate)
monkeypatch.setattr(
subprocess, 'Popen', fakesubprocesspopen)
pytest.raises(Exception, vss.stop_sql_server(backup_opt))
def test_vss_create_shadow_copy(self, monkeypatch):
fake_disable_redirection = FakeDisableFileSystemRedirection()
fakelogging = FakeLogging()
@ -173,4 +103,4 @@ class TestVss:
monkeypatch.setattr(
subprocess, 'Popen', fakesubprocesspopen)
assert vss.vss_delete_shadow_copy('shadow_id', 'C:\\') is True
assert vss.vss_delete_shadow_copy('shadow_id', 'C:\\') is True

71
tests/test_winutils.py
View File

@ -17,7 +17,9 @@ from freezer.winutils import use_shadow
from freezer.winutils import clean_tar_command
from freezer.winutils import add_gzip_to_command
from freezer.winutils import DisableFileSystemRedirection
from freezer import winutils
from commons import *
import logging
import pytest
@ -33,7 +35,7 @@ class TestWinutils:
test_volume = 'C:'
test_volume2 = 'C:\\'
path = 'C:\\Users\\Test'
expected = 'C:\\shadowcopy\\Users\\Test'
expected = 'C:\\freezer_shadowcopy\\Users\\Test'
assert use_shadow(path, test_volume2) == expected
# test if the volume format is incorrect
@ -68,3 +70,70 @@ class TestWinutils:
pytest.raises(Exception, fake_disable_redirection.__enter__)
pytest.raises(Exception, fake_disable_redirection.__exit__)
def test_start_sql_server(self, monkeypatch):
fake_disable_redirection = FakeDisableFileSystemRedirection()
backup_opt = BackupOpt1()
fakelogging = FakeLogging()
fakesubprocess = FakeSubProcess()
fakesubprocesspopen = fakesubprocess.Popen()
monkeypatch.setattr(
subprocess.Popen, 'communicate',
fakesubprocesspopen.communicate)
monkeypatch.setattr(
subprocess, 'Popen', fakesubprocesspopen)
monkeypatch.setattr(
winutils.DisableFileSystemRedirection, '__enter__',
fake_disable_redirection.__enter__)
monkeypatch.setattr(
winutils.DisableFileSystemRedirection, '__exit__',
fake_disable_redirection.__exit__)
monkeypatch.setattr(logging, 'info', fakelogging.info)
assert winutils.start_sql_server(backup_opt) is not False
fakesubprocess = FakeSubProcess3()
fakesubprocesspopen = fakesubprocess.Popen()
monkeypatch.setattr(
subprocess.Popen, 'communicate',
fakesubprocesspopen.communicate)
monkeypatch.setattr(
subprocess, 'Popen', fakesubprocesspopen)
pytest.raises(Exception, winutils.start_sql_server(backup_opt))
def test_stop_sql_server(self, monkeypatch):
fake_disable_redirection = FakeDisableFileSystemRedirection()
backup_opt = BackupOpt1()
fakelogging = FakeLogging()
fakesubprocess = FakeSubProcess()
fakesubprocesspopen = fakesubprocess.Popen()
monkeypatch.setattr(
subprocess.Popen, 'communicate',
fakesubprocesspopen.communicate)
monkeypatch.setattr(
subprocess, 'Popen', fakesubprocesspopen)
monkeypatch.setattr(
winutils.DisableFileSystemRedirection, '__enter__',
fake_disable_redirection.__enter__)
monkeypatch.setattr(
winutils.DisableFileSystemRedirection, '__exit__',
fake_disable_redirection.__exit__)
monkeypatch.setattr(logging, 'info', fakelogging.info)
assert winutils.start_sql_server(backup_opt) is not False
fakesubprocess = FakeSubProcess3()
fakesubprocesspopen = fakesubprocess.Popen()
monkeypatch.setattr(
subprocess.Popen, 'communicate',
fakesubprocesspopen.communicate)
monkeypatch.setattr(
subprocess, 'Popen', fakesubprocesspopen)
pytest.raises(Exception, winutils.stop_sql_server(backup_opt))