openstack/freezer
Code Issues Proposed changes

add option to choose compression algorithm with choose between gzip, bzip2 and xz

Browse Source 
xz.exe with sources added for windows

Implements: blueprint compression-options

Change-Id: I62d7b0c696556c68ab7bab5cd00689f0ba731b9e
This commit is contained in:
Memo Garcia 2015-08-07 10:25:53 +01:00 committed by sbartel
parent f6c1c1db25
commit 309e6e1e81
13 changed files with 66 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
freezer/arguments.py
View File

@ -64,7 +64,9 @@ DEFAULT_PARAMS = {
'upload': True, 'mode': 'fs', 'action': 'backup',
'vssadmin': True, 'shadow': '', 'shadow_path': '',
'windows_volume': '', 'command': None, 'metadata_out': False,
'storage': 'swift', 'ssh_key': '', 'ssh_username': '', 'ssh_host': ''}
'storage': 'swift', 'ssh_key': '', 'ssh_username': '', 'ssh_host': '',
'compression': 'gzip'
}
def alter_proxy(args_dict):
@ -408,6 +410,11 @@ def backup_arguments(args_dict={}):
'--command', action='store',
help='Command executed by exec action',
dest='command', default=None)
arg_parser.add_argument(
'--compression', action='store',
choices=['gzip', 'bzip2', 'xz'],
help='compression algorithm to use. gzip is default algorithm',
dest='compression', default='gzip')
arg_parser.add_argument(
'--storage', action='store',

4
freezer/backup.py
View File

@ -287,9 +287,9 @@ def backup(backup_opt_dict, storage):
chdir_path = os.path.dirname(chdir_path)
os.chdir(chdir_path)
builder = TarCommandBuilder(backup_opt_dict.tar_path,
filepath=filepath)
filepath,
backup_opt_dict.compression)
builder.set_dereference(backup_opt_dict.dereference_symlink)
if backup_opt_dict.exclude:
builder.set_exclude(backup_opt_dict.exclude)
if backup_opt_dict.encrypt_pass_file:

BIN
freezer/bin/xz-5.2.1.tar.gz Normal file
View File

Binary file not shown.

BIN
freezer/bin/xz.exe Normal file
View File

Binary file not shown.

3
freezer/job.py
View File

@ -128,7 +128,8 @@ class RestoreJob(Job):
if conf.backup_media == 'fs':
storage = conf.storage
builder = tar.TarCommandRestoreBuilder(conf.tar_path,
restore_abs_path)
restore_abs_path,
conf.compression)
if conf.dry_run:
builder.set_dry_run()
if winutils.is_windows():

37
freezer/tar.py
View File

@ -22,7 +22,6 @@ Freezer Tar related functions
"""
from freezer import winutils
import os
import logging
import subprocess
@ -58,7 +57,7 @@ class TarCommandBuilder:
"""
COMMAND_TEMPLATE = (
"{gnutar_path} --create -z --warning=none --no-check-device "
"{gnutar_path} --create {algo} --warning=none --no-check-device "
"--one-file-system --preserve-permissions --same-owner "
"--seek --ignore-failed-read")
@ -69,7 +68,7 @@ class TarCommandBuilder:
'all': '--hard-dereference --dereference',
'none': ''}
def __init__(self, gnutar_path, filepath):
def __init__(self, gnutar_path, filepath, compression_algo):
self.dereference = 'none'
self.gnutar_path = gnutar_path
self.exclude = None
@ -83,6 +82,7 @@ class TarCommandBuilder:
self.ssh_key = None
self.ssh_user = None
self.ssh_ip = None
self.compression_algo = get_tar_flag_from_algo(compression_algo)
def set_output_file(self, output_file):
self.output_file = output_file
@ -126,9 +126,11 @@ class TarCommandBuilder:
def build(self):
tar_command = self.COMMAND_TEMPLATE.format(
gnutar_path=self.gnutar_path, dereference=self.dereference)
gnutar_path=self.gnutar_path, algo=self.compression_algo,
dereference=self.dereference)
if self.dereference:
"{0} {1}".format(tar_command, self.dereference)
if self.listed_incremental:
tar_command = self.LISTED_TEMPLATE.format(
tar_command=tar_command,
@ -158,14 +160,14 @@ class TarCommandBuilder:
class TarCommandRestoreBuilder:
WINDOWS_TEMPLATE = '{0} -x -z --incremental --unlink-first ' \
WINDOWS_TEMPLATE = '{0} -x {1} --incremental --unlink-first ' \
'--ignore-zeros -f - '
DRY_RUN_TEMPLATE = '{0} -z --incremental --list ' \
DRY_RUN_TEMPLATE = '{0} {1} --incremental --list ' \
'--ignore-zeros --warning=none'
NORMAL_TEMPLATE = '{0} -z --incremental --extract --unlink-first ' \
'--ignore-zeros --warning=none --overwrite --directory {1}'
NORMAL_TEMPLATE = '{0} {1} --incremental --extract --unlink-first ' \
'--ignore-zeros --warning=none --overwrite --directory {2}'
def __init__(self, tar_path, restore_path):
def __init__(self, tar_path, restore_path, compression_algo):
self.dry_run = False
self.is_windows = False
self.openssl_path = None
@ -176,6 +178,7 @@ class TarCommandRestoreBuilder:
self.ssh_key = None
self.ssh_user = None
self.ssh_ip = None
self.compression_algo = get_tar_flag_from_algo(compression_algo)
def set_dry_run(self):
self.dry_run = True
@ -197,11 +200,14 @@ class TarCommandRestoreBuilder:
def build(self):
if self.is_windows:
tar_command = self.NORMAL_TEMPLATE.format(self.tar_path)
tar_command = self.NORMAL_TEMPLATE.format(self.tar_path,
self.compression_algo)
elif self.dry_run:
tar_command = self.DRY_RUN_TEMPLATE.format(self.tar_path)
tar_command = self.DRY_RUN_TEMPLATE.format(self.tar_path,
self.compression_algo)
else:
tar_command = self.NORMAL_TEMPLATE.format(self.tar_path,
self.compression_algo,
self.restore_path)
if self.archive and not self.ssh_key:
@ -284,3 +290,12 @@ def tar_backup(path_to_backup, max_segment_size, tar_command, backup_queue):
if len(tar_chunk) < max_segment_size:
backup_queue.put(
({("%08d" % file_chunk_index): tar_chunk}))
def get_tar_flag_from_algo(compression):
algo = {
'gzip': '-z',
'bzip2': '-j',
'xz': '-J',
}
return algo.get(compression)

4
setup.py
View File

@ -109,6 +109,7 @@ setup(
('freezer/bin', ['freezer/bin/LICENSE']),
('freezer/bin', ['freezer/bin/gzip.exe']),
('freezer/bin', ['freezer/bin/tar.exe']),
('freezer/bin', ['freezer/bin/xz.exe']),
('freezer/bin', ['freezer/bin/cygwin1.dll']),
('freezer/bin', ['freezer/bin/cygintl-8.dll']),
('freezer/bin', ['freezer/bin/cygiconv-2.dll']),
@ -116,5 +117,6 @@ setup(
('freezer/bin', ['freezer/bin/bzip2-1.0.6.tar.gz']),
('freezer/bin', ['freezer/bin/gzip-1.6-1.src.tar']),
('freezer/bin', ['freezer/bin/tar-1.27.1.tar.xz']),
('freezer/bin', ['freezer/bin/findutils-4.5.12.tar.gz'])]
('freezer/bin', ['freezer/bin/findutils-4.5.12.tar.gz']),
('freezer/bin', ['freezer/bin/xz-5.2.1.tar.gz'])]
)

1
tests/commons.py
View File

@ -795,6 +795,7 @@ class BackupOpt1:
self.client_manager.get_nova = Mock(return_value=nova_client)
self.command = None
self.compression = 'gzip'
class FakeMySQLdb:

4
tests/test_backup.py
View File

@ -104,7 +104,7 @@ class TestBackUP:
assert storage.backup(
"/tmp/", "hostname_backup_name",
tar.TarCommandBuilder(tar_path(), ".")) is None
tar.TarCommandBuilder(tar_path(), ".", "gzip")) is None
backup_opt.__dict__['no_incremental'] = False
with open(
@ -112,7 +112,7 @@ class TestBackUP:
fd.write('testcontent\n')
assert storage.backup(
"/tmp/", "hostname_backup_name",
tar.TarCommandBuilder(tar_path(), ".")) is None
tar.TarCommandBuilder(tar_path(), ".", "gzip")) is None
def test_backup_mode_mongo(self, monkeypatch, tmpdir):

18
tests/test_local.py
View File

@ -47,7 +47,7 @@ class TestLocalStorage(object):
def test(self, tmpdir):
backup_dir, files_dir = self.create_dirs(tmpdir)
storage = local.LocalStorage(backup_dir)
builder = tar.TarCommandBuilder(commons.tar_path(), ".")
builder = tar.TarCommandBuilder(commons.tar_path(), ".", "gzip")
storage.backup(files_dir, "file_backup", builder)
storage.get_backups()
@ -68,7 +68,7 @@ class TestLocalStorage(object):
def test_get_backups(self, tmpdir):
backup_dir, files_dir = self.create_dirs(tmpdir)
storage = local.LocalStorage(backup_dir)
builder = tar.TarCommandBuilder(commons.tar_path(), ".")
builder = tar.TarCommandBuilder(commons.tar_path(), ".", "gzip")
os.chdir(files_dir)
storage.backup(files_dir, "file_backup", builder)
backups = storage.get_backups()
@ -77,7 +77,7 @@ class TestLocalStorage(object):
def test_incremental_backup(self, tmpdir):
backup_dir, files_dir = self.create_dirs(tmpdir)
storage = local.LocalStorage(backup_dir)
builder = tar.TarCommandBuilder(commons.tar_path(), ".")
builder = tar.TarCommandBuilder(commons.tar_path(), ".", "gzip")
os.chdir(files_dir)
storage.backup(files_dir, "file_backup", builder)
backups = storage.get_backups()
@ -89,7 +89,7 @@ class TestLocalStorage(object):
def test_incremental_restore(self, tmpdir):
backup_dir, files_dir = self.create_dirs(tmpdir)
storage = local.LocalStorage(backup_dir)
builder = tar.TarCommandBuilder(commons.tar_path(), ".")
builder = tar.TarCommandBuilder(commons.tar_path(), ".", "gzip")
os.chdir(files_dir)
storage.backup(files_dir, "file_backup", builder)
backups = storage.get_backups()
@ -102,7 +102,8 @@ class TestLocalStorage(object):
assert not os.listdir(files_dir)
utils.create_dir(files_dir)
backup = storage.get_backups()[0]
builder = tar.TarCommandRestoreBuilder(commons.tar_path(), files_dir)
builder = tar.TarCommandRestoreBuilder(commons.tar_path(), files_dir,
"gzip")
storage.restore(backup.latest_update, files_dir, builder)
files = os.listdir(files_dir)
assert len(files) == 2
@ -114,7 +115,7 @@ class TestLocalStorage(object):
def test_backup_file(self, tmpdir):
backup_dir, files_dir = self.create_dirs(tmpdir)
storage = local.LocalStorage(backup_dir)
builder = tar.TarCommandBuilder(commons.tar_path(), "file_1")
builder = tar.TarCommandBuilder(commons.tar_path(), "file_1", "gzip")
os.chdir(files_dir)
storage.backup(files_dir + "/file_1", "file_backup", builder)
for path in os.listdir(files_dir):
@ -122,7 +123,8 @@ class TestLocalStorage(object):
assert not os.listdir(files_dir)
utils.create_dir(files_dir)
backup = storage.get_backups()[0]
builder = tar.TarCommandRestoreBuilder(commons.tar_path(), files_dir)
builder = tar.TarCommandRestoreBuilder(commons.tar_path(), files_dir,
"gzip")
storage.restore(backup, files_dir, builder)
files = os.listdir(files_dir)
assert len(files) == 1
@ -130,7 +132,7 @@ class TestLocalStorage(object):
def test_remove_backup(self, tmpdir):
backup_dir, files_dir = self.create_dirs(tmpdir)
storage = local.LocalStorage(backup_dir)
builder = tar.TarCommandBuilder(commons.tar_path(), ".")
builder = tar.TarCommandBuilder(commons.tar_path(), ".", "gzip")
os.chdir(files_dir)
storage.backup(files_dir, "file_backup", builder)
backups = storage.get_backups()

8
tests/test_storage.py
View File

@ -96,7 +96,7 @@ class TestBackup(unittest.TestCase):
storage.Backup("host_backup_f", 1000, 0),
last
]
builder = tar.TarCommandRestoreBuilder("", "")
builder = tar.TarCommandRestoreBuilder("", "", "gzip")
self.assertRaises(ValueError, t.restore_latest, "test", ".", builder)
t.restore = mock.Mock()
t.restore_latest("host_backup", ".", builder)
@ -116,7 +116,7 @@ class TestBackup(unittest.TestCase):
storage.Backup("host_backup_f", 1000, 0),
storage.Backup("host_backup", 5000, 0),
]
builder = tar.TarCommandRestoreBuilder("", "")
builder = tar.TarCommandRestoreBuilder("", "", "gzip")
t.restore = mock.Mock()
t.restore_latest("host_backup", ".", builder)
t.restore.assert_called_with(increment, ".", builder)
@ -134,7 +134,7 @@ class TestBackup(unittest.TestCase):
storage.Backup("host_backup", 5000, 0),
]
t.restore = mock.Mock()
builder = tar.TarCommandRestoreBuilder("", "")
builder = tar.TarCommandRestoreBuilder("", "", "gzip")
t.restore_from_date("host_backup", ".", builder, 3234)
t.restore.assert_called_with(backup_restore, ".", builder)
@ -151,7 +151,7 @@ class TestBackup(unittest.TestCase):
storage.Backup("host_backup", 5000, 0),
]
t.restore = mock.Mock()
builder = tar.TarCommandRestoreBuilder("", "")
builder = tar.TarCommandRestoreBuilder("", "", "gzip")
t.restore_from_date("host_backup", ".", builder, 3234)
t.restore.assert_called_with(increment, ".", builder)

7
tests/test_tar.py
View File

@ -22,7 +22,7 @@ Hudson (tjh@cryptsoft.com).
"""
from commons import *
from freezer.tar import (tar_restore, tar_backup)
from freezer.tar import (tar_restore, tar_backup, get_tar_flag_from_algo)
from freezer import winutils
import os
@ -108,3 +108,8 @@ class TestTar:
fakeos1 = Os1()
monkeypatch.setattr(os.path, 'exists', fakeos1.exists)
backup_opt.dry_run = False
def test_get_tar_flag_from_algo(self):
assert get_tar_flag_from_algo('gzip') == '-z'
assert get_tar_flag_from_algo('bzip2') == '-j'
assert get_tar_flag_from_algo('xz') == '-J'

4
tests/test_tar_builders.py
View File

@ -5,7 +5,7 @@ from freezer import tar
class TestTarCommandBuilder(unittest.TestCase):
def setUp(self):
self.builder = tar.TarCommandBuilder("gnutar", ".")
self.builder = tar.TarCommandBuilder("gnutar", ".", "gzip")
def test_build(self):
self.assertEquals(
@ -37,7 +37,7 @@ class TestTarCommandBuilder(unittest.TestCase):
class TestTarCommandRestoreBuilder(unittest.TestCase):
def setUp(self):
self.builder = tar.TarCommandRestoreBuilder("gnutar", "restore_path")
self.builder = tar.TarCommandRestoreBuilder("gnutar", "restore_path", "gzip")
def test(self):
self.assertEquals(