add option to choose compression algorithm with choose between gzip, bzip2 and xz

xz.exe with sources added for windows

Implements: blueprint compression-options

Change-Id: I62d7b0c696556c68ab7bab5cd00689f0ba731b9e

freezer/arguments.py

@@ -64,7 +64,9 @@ DEFAULT_PARAMS = {
     'upload': True, 'mode': 'fs', 'action': 'backup',
     'vssadmin': True, 'shadow': '', 'shadow_path': '',
     'windows_volume': '', 'command': None, 'metadata_out': False,
-    'storage': 'swift', 'ssh_key': '', 'ssh_username': '', 'ssh_host': ''}
+    'storage': 'swift', 'ssh_key': '', 'ssh_username': '', 'ssh_host': '',
+    'compression': 'gzip'
+}
 
 
 def alter_proxy(args_dict):
@@ -408,6 +410,11 @@ def backup_arguments(args_dict={}):
         '--command', action='store',
         help='Command executed by exec action',
         dest='command', default=None)
+    arg_parser.add_argument(
+        '--compression', action='store',
+        choices=['gzip', 'bzip2', 'xz'],
+        help='compression algorithm to use. gzip is default algorithm',
+        dest='compression', default='gzip')
 
     arg_parser.add_argument(
         '--storage', action='store',

freezer/backup.py

@@ -287,9 +287,9 @@ def backup(backup_opt_dict, storage):
                 chdir_path = os.path.dirname(chdir_path)
             os.chdir(chdir_path)
             builder = TarCommandBuilder(backup_opt_dict.tar_path,
-                                        filepath=filepath)
+                                        filepath,
+                                        backup_opt_dict.compression)
             builder.set_dereference(backup_opt_dict.dereference_symlink)
-
             if backup_opt_dict.exclude:
                 builder.set_exclude(backup_opt_dict.exclude)
             if backup_opt_dict.encrypt_pass_file:

freezer/job.py

@@ -128,7 +128,8 @@ class RestoreJob(Job):
         if conf.backup_media == 'fs':
             storage = conf.storage
             builder = tar.TarCommandRestoreBuilder(conf.tar_path,
-                                                   restore_abs_path)
+                                                   restore_abs_path,
+                                                   conf.compression)
             if conf.dry_run:
                 builder.set_dry_run()
             if winutils.is_windows():

freezer/tar.py

@@ -22,7 +22,6 @@ Freezer Tar related functions
 """
 
 from freezer import winutils
-
 import os
 import logging
 import subprocess
@@ -58,7 +57,7 @@ class TarCommandBuilder:
     """
 
     COMMAND_TEMPLATE = (
-        "{gnutar_path} --create -z --warning=none --no-check-device "
+        "{gnutar_path} --create {algo} --warning=none --no-check-device "
         "--one-file-system --preserve-permissions --same-owner "
         "--seek --ignore-failed-read")
 
@@ -69,7 +68,7 @@ class TarCommandBuilder:
                         'all': '--hard-dereference --dereference',
                         'none': ''}
 
-    def __init__(self, gnutar_path, filepath):
+    def __init__(self, gnutar_path, filepath, compression_algo):
         self.dereference = 'none'
         self.gnutar_path = gnutar_path
         self.exclude = None
@@ -83,6 +82,7 @@ class TarCommandBuilder:
         self.ssh_key = None
         self.ssh_user = None
         self.ssh_ip = None
+        self.compression_algo = get_tar_flag_from_algo(compression_algo)
 
     def set_output_file(self, output_file):
         self.output_file = output_file
@@ -126,9 +126,11 @@ class TarCommandBuilder:
 
     def build(self):
         tar_command = self.COMMAND_TEMPLATE.format(
-            gnutar_path=self.gnutar_path, dereference=self.dereference)
+            gnutar_path=self.gnutar_path, algo=self.compression_algo,
+            dereference=self.dereference)
         if self.dereference:
             "{0} {1}".format(tar_command, self.dereference)
+
         if self.listed_incremental:
             tar_command = self.LISTED_TEMPLATE.format(
                 tar_command=tar_command,
@@ -158,14 +160,14 @@ class TarCommandBuilder:
 
 
 class TarCommandRestoreBuilder:
-    WINDOWS_TEMPLATE = '{0} -x -z --incremental --unlink-first ' \
+    WINDOWS_TEMPLATE = '{0} -x {1} --incremental --unlink-first ' \
         '--ignore-zeros -f - '
-    DRY_RUN_TEMPLATE = '{0} -z --incremental --list ' \
+    DRY_RUN_TEMPLATE = '{0} {1} --incremental --list ' \
         '--ignore-zeros --warning=none'
-    NORMAL_TEMPLATE = '{0} -z --incremental --extract --unlink-first ' \
+    NORMAL_TEMPLATE = '{0} {1} --incremental --extract --unlink-first ' \
-        '--ignore-zeros --warning=none --overwrite --directory {1}'
+        '--ignore-zeros --warning=none --overwrite --directory {2}'
 
-    def __init__(self, tar_path, restore_path):
+    def __init__(self, tar_path, restore_path, compression_algo):
         self.dry_run = False
         self.is_windows = False
         self.openssl_path = None
@@ -176,6 +178,7 @@ class TarCommandRestoreBuilder:
         self.ssh_key = None
         self.ssh_user = None
         self.ssh_ip = None
+        self.compression_algo = get_tar_flag_from_algo(compression_algo)
 
     def set_dry_run(self):
         self.dry_run = True
@@ -197,11 +200,14 @@ class TarCommandRestoreBuilder:
 
     def build(self):
         if self.is_windows:
-            tar_command = self.NORMAL_TEMPLATE.format(self.tar_path)
+            tar_command = self.NORMAL_TEMPLATE.format(self.tar_path,
+                                                      self.compression_algo)
         elif self.dry_run:
-            tar_command = self.DRY_RUN_TEMPLATE.format(self.tar_path)
+            tar_command = self.DRY_RUN_TEMPLATE.format(self.tar_path,
+                                                       self.compression_algo)
         else:
             tar_command = self.NORMAL_TEMPLATE.format(self.tar_path,
+                                                      self.compression_algo,
                                                       self.restore_path)
 
         if self.archive and not self.ssh_key:
@@ -284,3 +290,12 @@ def tar_backup(path_to_backup, max_segment_size, tar_command, backup_queue):
     if len(tar_chunk) < max_segment_size:
         backup_queue.put(
             ({("%08d" % file_chunk_index): tar_chunk}))
+
+
+def get_tar_flag_from_algo(compression):
+    algo = {
+        'gzip': '-z',
+        'bzip2': '-j',
+        'xz': '-J',
+    }
+    return algo.get(compression)

setup.py

@@ -109,6 +109,7 @@ setup(
                 ('freezer/bin', ['freezer/bin/LICENSE']),
                 ('freezer/bin', ['freezer/bin/gzip.exe']),
                 ('freezer/bin', ['freezer/bin/tar.exe']),
+                ('freezer/bin', ['freezer/bin/xz.exe']),
                 ('freezer/bin', ['freezer/bin/cygwin1.dll']),
                 ('freezer/bin', ['freezer/bin/cygintl-8.dll']),
                 ('freezer/bin', ['freezer/bin/cygiconv-2.dll']),
@@ -116,5 +117,6 @@ setup(
                 ('freezer/bin', ['freezer/bin/bzip2-1.0.6.tar.gz']),
                 ('freezer/bin', ['freezer/bin/gzip-1.6-1.src.tar']),
                 ('freezer/bin', ['freezer/bin/tar-1.27.1.tar.xz']),
-                ('freezer/bin', ['freezer/bin/findutils-4.5.12.tar.gz'])]
+                ('freezer/bin', ['freezer/bin/findutils-4.5.12.tar.gz']),
+                ('freezer/bin', ['freezer/bin/xz-5.2.1.tar.gz'])]
 )

tests/commons.py

@@ -795,6 +795,7 @@ class BackupOpt1:
 
         self.client_manager.get_nova = Mock(return_value=nova_client)
         self.command = None
+        self.compression = 'gzip'
 
 
 class FakeMySQLdb:

tests/test_backup.py

@@ -104,7 +104,7 @@ class TestBackUP:
 
         assert storage.backup(
             "/tmp/", "hostname_backup_name",
-            tar.TarCommandBuilder(tar_path(), ".")) is None
+            tar.TarCommandBuilder(tar_path(), ".", "gzip")) is None
 
         backup_opt.__dict__['no_incremental'] = False
         with open(
@@ -112,7 +112,7 @@ class TestBackUP:
             fd.write('testcontent\n')
         assert storage.backup(
             "/tmp/", "hostname_backup_name",
-            tar.TarCommandBuilder(tar_path(), ".")) is None
+            tar.TarCommandBuilder(tar_path(), ".", "gzip")) is None
 
     def test_backup_mode_mongo(self, monkeypatch, tmpdir):
 

tests/test_local.py

@@ -47,7 +47,7 @@ class TestLocalStorage(object):
     def test(self, tmpdir):
         backup_dir, files_dir = self.create_dirs(tmpdir)
         storage = local.LocalStorage(backup_dir)
-        builder = tar.TarCommandBuilder(commons.tar_path(), ".")
+        builder = tar.TarCommandBuilder(commons.tar_path(), ".", "gzip")
         storage.backup(files_dir, "file_backup", builder)
         storage.get_backups()
 
@@ -68,7 +68,7 @@ class TestLocalStorage(object):
     def test_get_backups(self, tmpdir):
         backup_dir, files_dir = self.create_dirs(tmpdir)
         storage = local.LocalStorage(backup_dir)
-        builder = tar.TarCommandBuilder(commons.tar_path(), ".")
+        builder = tar.TarCommandBuilder(commons.tar_path(), ".", "gzip")
         os.chdir(files_dir)
         storage.backup(files_dir, "file_backup", builder)
         backups = storage.get_backups()
@@ -77,7 +77,7 @@ class TestLocalStorage(object):
     def test_incremental_backup(self, tmpdir):
         backup_dir, files_dir = self.create_dirs(tmpdir)
         storage = local.LocalStorage(backup_dir)
-        builder = tar.TarCommandBuilder(commons.tar_path(), ".")
+        builder = tar.TarCommandBuilder(commons.tar_path(), ".", "gzip")
         os.chdir(files_dir)
         storage.backup(files_dir, "file_backup", builder)
         backups = storage.get_backups()
@@ -89,7 +89,7 @@ class TestLocalStorage(object):
     def test_incremental_restore(self, tmpdir):
         backup_dir, files_dir = self.create_dirs(tmpdir)
         storage = local.LocalStorage(backup_dir)
-        builder = tar.TarCommandBuilder(commons.tar_path(), ".")
+        builder = tar.TarCommandBuilder(commons.tar_path(), ".", "gzip")
         os.chdir(files_dir)
         storage.backup(files_dir, "file_backup", builder)
         backups = storage.get_backups()
@@ -102,7 +102,8 @@ class TestLocalStorage(object):
         assert not os.listdir(files_dir)
         utils.create_dir(files_dir)
         backup = storage.get_backups()[0]
-        builder = tar.TarCommandRestoreBuilder(commons.tar_path(), files_dir)
+        builder = tar.TarCommandRestoreBuilder(commons.tar_path(), files_dir,
+                                               "gzip")
         storage.restore(backup.latest_update, files_dir, builder)
         files = os.listdir(files_dir)
         assert len(files) == 2
@@ -114,7 +115,7 @@ class TestLocalStorage(object):
     def test_backup_file(self, tmpdir):
         backup_dir, files_dir = self.create_dirs(tmpdir)
         storage = local.LocalStorage(backup_dir)
-        builder = tar.TarCommandBuilder(commons.tar_path(), "file_1")
+        builder = tar.TarCommandBuilder(commons.tar_path(), "file_1", "gzip")
         os.chdir(files_dir)
         storage.backup(files_dir + "/file_1", "file_backup", builder)
         for path in os.listdir(files_dir):
@@ -122,7 +123,8 @@ class TestLocalStorage(object):
         assert not os.listdir(files_dir)
         utils.create_dir(files_dir)
         backup = storage.get_backups()[0]
-        builder = tar.TarCommandRestoreBuilder(commons.tar_path(), files_dir)
+        builder = tar.TarCommandRestoreBuilder(commons.tar_path(), files_dir,
+                                               "gzip")
         storage.restore(backup, files_dir, builder)
         files = os.listdir(files_dir)
         assert len(files) == 1
@@ -130,7 +132,7 @@ class TestLocalStorage(object):
     def test_remove_backup(self, tmpdir):
         backup_dir, files_dir = self.create_dirs(tmpdir)
         storage = local.LocalStorage(backup_dir)
-        builder = tar.TarCommandBuilder(commons.tar_path(), ".")
+        builder = tar.TarCommandBuilder(commons.tar_path(), ".", "gzip")
         os.chdir(files_dir)
         storage.backup(files_dir, "file_backup", builder)
         backups = storage.get_backups()

tests/test_storage.py

@@ -96,7 +96,7 @@ class TestBackup(unittest.TestCase):
             storage.Backup("host_backup_f", 1000, 0),
             last
         ]
-        builder = tar.TarCommandRestoreBuilder("", "")
+        builder = tar.TarCommandRestoreBuilder("", "", "gzip")
         self.assertRaises(ValueError, t.restore_latest, "test", ".", builder)
         t.restore = mock.Mock()
         t.restore_latest("host_backup", ".", builder)
@@ -116,7 +116,7 @@ class TestBackup(unittest.TestCase):
             storage.Backup("host_backup_f", 1000, 0),
             storage.Backup("host_backup", 5000, 0),
         ]
-        builder = tar.TarCommandRestoreBuilder("", "")
+        builder = tar.TarCommandRestoreBuilder("", "", "gzip")
         t.restore = mock.Mock()
         t.restore_latest("host_backup", ".", builder)
         t.restore.assert_called_with(increment, ".", builder)
@@ -134,7 +134,7 @@ class TestBackup(unittest.TestCase):
             storage.Backup("host_backup", 5000, 0),
         ]
         t.restore = mock.Mock()
-        builder = tar.TarCommandRestoreBuilder("", "")
+        builder = tar.TarCommandRestoreBuilder("", "", "gzip")
         t.restore_from_date("host_backup", ".", builder, 3234)
         t.restore.assert_called_with(backup_restore, ".", builder)
 
@@ -151,7 +151,7 @@ class TestBackup(unittest.TestCase):
             storage.Backup("host_backup", 5000, 0),
         ]
         t.restore = mock.Mock()
-        builder = tar.TarCommandRestoreBuilder("", "")
+        builder = tar.TarCommandRestoreBuilder("", "", "gzip")
         t.restore_from_date("host_backup", ".", builder, 3234)
         t.restore.assert_called_with(increment, ".", builder)
 

tests/test_tar.py

@@ -22,7 +22,7 @@ Hudson (tjh@cryptsoft.com).
 """
 
 from commons import *
-from freezer.tar import (tar_restore, tar_backup)
+from freezer.tar import (tar_restore, tar_backup,  get_tar_flag_from_algo)
 from freezer import winutils
 
 import os
@@ -108,3 +108,8 @@ class TestTar:
         fakeos1 = Os1()
         monkeypatch.setattr(os.path, 'exists', fakeos1.exists)
         backup_opt.dry_run = False
+
+    def test_get_tar_flag_from_algo(self):
+        assert get_tar_flag_from_algo('gzip') == '-z'
+        assert get_tar_flag_from_algo('bzip2') == '-j'
+        assert get_tar_flag_from_algo('xz') == '-J'

tests/test_tar_builders.py

@@ -5,7 +5,7 @@ from freezer import tar
 class TestTarCommandBuilder(unittest.TestCase):
 
     def setUp(self):
-        self.builder = tar.TarCommandBuilder("gnutar", ".")
+        self.builder = tar.TarCommandBuilder("gnutar", ".", "gzip")
 
     def test_build(self):
         self.assertEquals(
@@ -37,7 +37,7 @@ class TestTarCommandBuilder(unittest.TestCase):
 
 class TestTarCommandRestoreBuilder(unittest.TestCase):
     def setUp(self):
-        self.builder = tar.TarCommandRestoreBuilder("gnutar", "restore_path")
+        self.builder = tar.TarCommandRestoreBuilder("gnutar", "restore_path", "gzip")
 
     def test(self):
         self.assertEquals(