Refactoring of code:

1) new snapshot module with code vss,
lvm and part of backup.py -
create_snapshot and remove_snapshot

This module should contain all code related shadow,
vss and any new snapshots.

2) new openstack module with code from osclient,
restore and backup (BackupOS)

This module should contain code related to cinder/nova backups

3) new utils module with code from utils, win_utils,
bandwidth, config, exec_cmd, validator, streaming

This module should containd code related to anything
that cannot be extracted to its own module

4) new test modules - snapshot, utils, openstack and apiclient.

Test scheduler daemon is now located in scheduler module.

5) Backup.py is deleted and code placed to
openstack.backup (BackupOS) and to job.BackupJob

Change-Id: I46194b40b3a65c15b6095cfb7388dcfd30d95997

freezer/apiclient/client.py

@@ -29,7 +29,7 @@ from freezer.apiclient import backups
 from freezer.apiclient import jobs
 from freezer.apiclient import registration
 from freezer.apiclient import sessions
-from freezer.utils import Namespace
+from freezer.utils import utils
 
 CONF = cfg.CONF
 
@@ -209,7 +209,7 @@ class Client(object):
         # this creates a namespace for self.opts when the client is
         # created from other method rather than command line arguments.
         if self.opts is None:
-            self.opts = Namespace({})
+            self.opts = utils.Namespace({})
         if token:
             self.opts.os_token = token
         if username:

freezer/common/config.py

@@ -23,9 +23,9 @@ import sys
 from tempfile import NamedTemporaryFile
 
 from freezer import __version__ as FREEZER_VERSION
-from freezer import config as freezer_config
-from freezer import utils
-from freezer import winutils
+from freezer.utils import config as freezer_config
+from freezer.utils import utils
+from freezer.utils import winutils
 
 
 CONF = cfg.CONF

freezer/engine/engine.py

@@ -18,11 +18,12 @@ Freezer general utils functions
 import multiprocessing
 import time
 
-from freezer import streaming
-from freezer import utils
 from oslo_config import cfg
 from oslo_log import log
 
+from freezer.utils import streaming
+from freezer.utils import utils
+
 CONF = cfg.CONF
 logging = log.getLogger(__name__)
 

freezer/engine/tar/tar_builders.py

@@ -15,8 +15,7 @@ limitations under the License.
 
 Freezer Tar related functions
 """
-from freezer import utils
-from freezer.utils import get_executable_path
+from freezer.utils import utils
 
 
 class TarCommandBuilder:
@@ -157,7 +156,7 @@ def get_tar_flag_from_algo(compression):
         'bzip2': '-j',
         'xz': '-J',
     }
-    compression_exec = get_executable_path(compression)
+    compression_exec = utils.get_executable_path(compression)
     if not compression_exec:
         raise Exception("[*] Critical Error: [*] {0} executable not found ".
                         format(compression))

freezer/engine/tar/tar_engine.py

@@ -19,12 +19,10 @@ import logging
 import os
 import subprocess
 import sys
-import threading
 
 from freezer.engine import engine
 from freezer.engine.tar import tar_builders
-from freezer import streaming
-from freezer import winutils
+from freezer.utils import winutils
 
 
 class TarBackupEngine(engine.BackupEngine):

freezer/job.py

@@ -16,14 +16,16 @@ limitations under the License.
 """
 
 import datetime
+import os
 from oslo_utils import importutils
 import sys
 import time
 
-from freezer import backup
-from freezer import exec_cmd
-from freezer import restore
-from freezer import utils
+from freezer.openstack import backup
+from freezer.openstack import restore
+from freezer.snapshot import snapshot
+from freezer.utils import exec_cmd
+from freezer.utils import utils
 
 from oslo_config import cfg
 from oslo_log import log
@@ -84,8 +86,7 @@ class BackupJob(Job):
         mod_name = 'freezer.mode.{0}.{1}'.format(
             self.conf.mode, self.conf.mode.capitalize() + 'Mode')
         app_mode = importutils.import_object(mod_name, self.conf)
-        backup_instance = backup.backup(
-            self.conf, self.storage, self.engine, app_mode)
+        backup_instance = self.backup(app_mode)
 
         level = backup_instance.level if backup_instance else 0
 
@@ -110,6 +111,65 @@ class BackupJob(Job):
             metadata[field_name] = self.conf.__dict__.get(field_name, '') or ''
         return metadata
 
+    def backup(self, app_mode):
+        """
+
+        :type app_mode: freezer.mode.mode.Mode
+        :return:
+        """
+        backup_media = self.conf.backup_media
+
+        time_stamp = utils.DateTime.now().timestamp
+        self.conf.time_stamp = time_stamp
+
+        if backup_media == 'fs':
+            app_mode.prepare()
+            snapshot_taken = snapshot.snapshot_create(self.conf)
+            if snapshot_taken:
+                app_mode.release()
+            try:
+                filepath = '.'
+                chdir_path = os.path.expanduser(
+                    os.path.normpath(self.conf.path_to_backup.strip()))
+                if not os.path.isdir(chdir_path):
+                    filepath = os.path.basename(chdir_path)
+                    chdir_path = os.path.dirname(chdir_path)
+                os.chdir(chdir_path)
+                hostname_backup_name = self.conf.hostname_backup_name
+                backup_instance = self.storage.create_backup(
+                    hostname_backup_name,
+                    self.conf.no_incremental,
+                    self.conf.max_level,
+                    self.conf.always_level,
+                    self.conf.restart_always_level,
+                    time_stamp=time_stamp)
+                self.engine.backup(filepath, backup_instance)
+                return backup_instance
+            finally:
+                # whether an error occurred or not, remove the snapshot anyway
+                app_mode.release()
+                if snapshot_taken:
+                    snapshot.snapshot_remove(
+                        self.conf, self.conf.shadow,
+                        self.conf.windows_volume)
+
+        backup_os = backup.BackupOs(self.conf.client_manager,
+                                    self.conf.container,
+                                    self.storage)
+
+        if backup_media == 'nova':
+            logging.info('[*] Executing nova backup')
+            backup_os.backup_nova(self.conf.nova_inst_id)
+        elif backup_media == 'cindernative':
+            logging.info('[*] Executing cinder backup')
+            backup_os.backup_cinder(self.conf.cindernative_vol_id)
+        elif backup_media == 'cinder':
+            logging.info('[*] Executing cinder snapshot')
+            backup_os.backup_cinder_by_glance(self.conf.cinder_vol_id)
+        else:
+            raise Exception('unknown parameter backup_media %s' % backup_media)
+        return None
+
 
 class RestoreJob(Job):
     @Job.executemethod

freezer/main.py

@@ -17,24 +17,26 @@ Freezer main execution function
 """
 import json
 import os
-from oslo_config import cfg
-from oslo_log import log
 import subprocess
 import sys
 
-from freezer import bandwidth
+from oslo_config import cfg
+from oslo_log import log
+
 from freezer.common import config as freezer_config
-from freezer import config
 from freezer.engine.tar import tar_engine
 from freezer import job
-from freezer import osclients
+from freezer.openstack import openstack
+from freezer.openstack import osclients
 from freezer.storage import local
 from freezer.storage import multiple
 from freezer.storage import ssh
 from freezer.storage import swift
-from freezer import utils
-from freezer import validator
-from freezer import winutils
+from freezer.utils import bandwidth
+from freezer.utils import config
+from freezer.utils import utils
+from freezer.utils import validator
+from freezer.utils import winutils
 
 CONF = cfg.CONF
 LOG = log.getLogger(__name__)
@@ -43,29 +45,12 @@ LOG = log.getLogger(__name__)
 def freezer_main(backup_args):
     """Freezer main loop for job execution.
     """
-    def set_max_process_priority():
-        """ Set freezer in max priority on the os """
-        # children processes inherit niceness from father
-        try:
-            LOG.warning(
-                '[*] Setting freezer execution with high CPU and I/O priority')
-            PID = os.getpid()
-            # Set cpu priority
-            os.nice(-19)
-            # Set I/O Priority to Real Time class with level 0
-            subprocess.call([
-                u'{0}'.format(utils.find_executable("ionice")),
-                u'-c', u'1', u'-n', u'0', u'-t',
-                u'-p', u'{0}'.format(PID)
-            ])
-        except Exception as priority_error:
-            LOG.warning('[*] Priority: {0}'.format(priority_error))
 
     if not backup_args.quiet:
         LOG.info('log file at {0}'.format(CONF.get('log_file')))
 
     if backup_args.max_priority:
-        set_max_process_priority()
+        utils.set_max_process_priority()
 
     bandwidth.monkeypatch_socket_bandwidth(backup_args)
 
@@ -156,10 +141,10 @@ def storage_from_dict(backup_args, work_dir, max_segment_size,
     container = backup_args['container']
     if storage_name == "swift":
         if "osrc" in backup_args:
-            options = utils.OpenstackOptions.create_from_dict(
+            options = openstack.OpenstackOptions.create_from_dict(
                 parse_osrc(backup_args['osrc']))
         else:
-            options = utils.OpenstackOptions.create_from_env()
+            options = openstack.OpenstackOptions.create_from_env()
         identity_api_version = (os_identity_api_version or
                                 options.identity_api_version)
         client_manager = osclients.ClientManager(
@@ -195,10 +180,5 @@ def main():
             CONF.print_help()
             sys.exit(1)
         freezer_main(backup_args)
-    except ValueError as err:
-        return fail(1, err, backup_args.quiet)
-    except ImportError as err:
-        return fail(1, err, backup_args.quiet)
     except Exception as err:
         return fail(1, err, backup_args.quiet)
-

freezer/mode/mysql.py

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from freezer import config
 from freezer.mode import mode
+from freezer.utils import config
 
 
 class MysqlMode(mode.Mode):

freezer/mode/sqlserver.py

@@ -14,10 +14,10 @@
 
 import logging
 
-from freezer import config
 from freezer.mode import mode
-from freezer import utils
-from freezer import winutils
+from freezer.utils import config
+from freezer.utils import utils
+from freezer.utils import winutils
 
 
 class SqlserverMode(mode.Mode):

freezer/openstack/backup.py

@@ -18,13 +18,11 @@ Freezer Backup modes related functions
 import os
 import time
 
-from freezer import lvm
-from freezer import utils
-from freezer import vss
-from freezer import winutils
 from oslo_config import cfg
 from oslo_log import log
 
+from freezer.utils import utils
+
 CONF = cfg.CONF
 logging = log.getLogger(__name__)
 home = os.path.expanduser("~")
@@ -118,99 +116,3 @@ class BackupOs:
         cinder = client_manager.get_cinder()
         cinder.backups.create(volume_id, self.container, name, description,
                               incremental=True, force=True)
-
-
-def snapshot_create(backup_opt_dict):
-    """
-    Calls the code to take fs snapshots, depending on the platform
-
-    :param backup_opt_dict:
-    :return: boolean value, True if snapshot has been taken, false otherwise
-    """
-
-    if winutils.is_windows():
-        if backup_opt_dict.snapshot:
-            # Create a shadow copy.
-            backup_opt_dict.shadow_path, backup_opt_dict.shadow = \
-                vss.vss_create_shadow_copy(backup_opt_dict.windows_volume)
-
-            backup_opt_dict.path_to_backup = winutils.use_shadow(
-                backup_opt_dict.path_to_backup,
-                backup_opt_dict.windows_volume)
-            return True
-        return False
-    else:
-        return lvm.lvm_snap(backup_opt_dict)
-
-
-def snapshot_remove(backup_opt_dict, shadow, windows_volume):
-    if winutils.is_windows():
-        # Delete the shadow copy after the backup
-        vss.vss_delete_shadow_copy(shadow, windows_volume)
-    else:
-        # Unmount and remove lvm snapshot volume
-        lvm.lvm_snap_remove(backup_opt_dict)
-
-
-def backup(backup_opt_dict, storage, engine, app_mode):
-    """
-
-    :param backup_opt_dict:
-    :param storage:
-    :type storage: freezer.storage.base.Storage
-    :param engine: Backup Engine
-    :type engine: freezer.engine.engine.BackupEngine
-    :type app_mode: freezer.mode.mode.Mode
-    :return:
-    """
-    backup_media = backup_opt_dict.backup_media
-
-    time_stamp = utils.DateTime.now().timestamp
-    backup_opt_dict.time_stamp = time_stamp
-
-    if backup_media == 'fs':
-        app_mode.prepare()
-        snapshot_taken = snapshot_create(backup_opt_dict)
-        if snapshot_taken:
-            app_mode.release()
-        try:
-            filepath = '.'
-            chdir_path = os.path.expanduser(
-                os.path.normpath(backup_opt_dict.path_to_backup.strip()))
-            if not os.path.isdir(chdir_path):
-                filepath = os.path.basename(chdir_path)
-                chdir_path = os.path.dirname(chdir_path)
-            os.chdir(chdir_path)
-            hostname_backup_name = backup_opt_dict.hostname_backup_name
-            backup_instance = storage.create_backup(
-                hostname_backup_name,
-                backup_opt_dict.no_incremental,
-                backup_opt_dict.max_level,
-                backup_opt_dict.always_level,
-                backup_opt_dict.restart_always_level,
-                time_stamp=time_stamp)
-            engine.backup(filepath, backup_instance)
-            return backup_instance
-        finally:
-            # whether an error occurred or not, remove the snapshot anyway
-            app_mode.release()
-            if snapshot_taken:
-                snapshot_remove(backup_opt_dict, backup_opt_dict.shadow,
-                                backup_opt_dict.windows_volume)
-
-    backup_os = BackupOs(backup_opt_dict.client_manager,
-                         backup_opt_dict.container,
-                         storage)
-
-    if backup_media == 'nova':
-        logging.info('[*] Executing nova backup')
-        backup_os.backup_nova(backup_opt_dict.nova_inst_id)
-    elif backup_media == 'cindernative':
-        logging.info('[*] Executing cinder backup')
-        backup_os.backup_cinder(backup_opt_dict.cindernative_vol_id)
-    elif backup_media == 'cinder':
-        logging.info('[*] Executing cinder snapshot')
-        backup_os.backup_cinder_by_glance(backup_opt_dict.cinder_vol_id)
-    else:
-        raise Exception('unknown parameter backup_media %s' % backup_media)
-    return None

freezer/openstack/openstack.py

@@ -0,0 +1,81 @@
+"""
+(c) Copyright 2015,2016 Hewlett-Packard Development Company, L.P.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import os
+
+
+class OpenstackOptions:
+    """
+    Stores credentials for OpenStack API.
+    Can be created using
+    >> create_from_env()
+    or
+    >> create_from_dict(dict)
+    """
+    def __init__(self, user_name, tenant_name, project_name, auth_url,
+                 password, identity_api_version, tenant_id=None,
+                 region_name=None, endpoint_type=None, cert=None,
+                 insecure=False, verify=True):
+        self.user_name = user_name
+        self.tenant_name = tenant_name
+        self.auth_url = auth_url
+        self.password = password
+        self.tenant_id = tenant_id
+        self.project_name = project_name
+        self.identity_api_version = identity_api_version
+        self.region_name = region_name
+        self.endpoint_type = endpoint_type
+        self.cert = cert
+        self.insecure = insecure
+        self.verify = verify
+        if not (self.password and self.user_name and self.auth_url and
+           (self.tenant_name or self.project_name)):
+            raise Exception("Please set up in your env:"
+                            "OS_USERNAME, OS_TENANT_NAME/OS_PROJECT_NAME,"
+                            " OS_AUTH_URL, OS_PASSWORD")
+
+    @property
+    def os_options(self):
+        """
+        :return: The OpenStack options which can have tenant_id,
+                 auth_token, service_type, endpoint_type, tenant_name,
+                 object_storage_url, region_name
+        """
+        return {'tenant_id': self.tenant_id,
+                'tenant_name': self.tenant_name,
+                'project_name': self.project_name,
+                'identity_api_version': self.identity_api_version,
+                'region_name': self.region_name,
+                'endpoint_type': self.endpoint_type}
+
+    @staticmethod
+    def create_from_env():
+        return OpenstackOptions.create_from_dict(os.environ)
+
+    @staticmethod
+    def create_from_dict(src_dict):
+        return OpenstackOptions(
+            user_name=src_dict.get('OS_USERNAME', None),
+            tenant_name=src_dict.get('OS_TENANT_NAME', None),
+            project_name=src_dict.get('OS_PROJECT_NAME', None),
+            auth_url=src_dict.get('OS_AUTH_URL', None),
+            identity_api_version=src_dict.get('OS_IDENTITY_API_VERSION',
+                                              '2.0'),
+            password=src_dict.get('OS_PASSWORD', None),
+            tenant_id=src_dict.get('OS_TENANT_ID', None),
+            region_name=src_dict.get('OS_REGION_NAME', None),
+            endpoint_type=src_dict.get('OS_ENDPOINT_TYPE', None),
+            cert=src_dict.get('OS_CERT', None)
+        )

freezer/openstack/osclients.py

@@ -17,11 +17,12 @@ import time
 from cinderclient import client as cclient
 from glanceclient import client as gclient
 from novaclient import client as nclient
-import swiftclient
-
-from freezer import utils
 from oslo_config import cfg
 from oslo_log import log
+import swiftclient
+
+from freezer.utils import utils
+
 
 CONF = cfg.CONF
 logging = log.getLogger(__name__)
@@ -39,7 +40,7 @@ class ClientManager:
         """
         Creates manager of connections to swift, nova, glance and cinder
         :param options: OpenstackOptions
-        :type options: freezer.utils.OpenstackOptions
+        :type options: freezer.openstack.openstack.OpenstackOptions
         :param insecure:
         :param swift_auth_version:
         :param dry_run:

freezer/openstack/restore.py

@@ -16,10 +16,11 @@ limitations under the License.
 Freezer restore modes related functions
 """
 
-from freezer import utils
 from oslo_config import cfg
 from oslo_log import log
 
+from freezer.utils import utils
+
 CONF = cfg.CONF
 logging = log.getLogger(__name__)
 

freezer/scheduler/arguments.py

@@ -20,7 +20,7 @@ import sys
 
 from freezer import __version__ as FREEZER_VERSION
 from freezer.apiclient import client as api_client
-from freezer import winutils
+from freezer.utils import winutils
 from oslo_config import cfg
 from oslo_log import log
 

freezer/scheduler/daemon.py

@@ -26,7 +26,7 @@ from time import sleep
 
 from freezer.lib.pep3143daemon import DaemonContext
 from freezer.lib.pep3143daemon import PidFile
-from freezer.utils import create_dir
+from freezer.utils import utils
 
 
 def setup_logging(log_file):
@@ -38,7 +38,7 @@ def setup_logging(log_file):
     def configure_logging(file_name):
         expanded_file_name = os.path.expanduser(file_name)
         expanded_dir_name = os.path.dirname(expanded_file_name)
-        create_dir(expanded_dir_name, do_log=False)
+        utils.create_dir(expanded_dir_name, do_log=False)
         logging.basicConfig(
             filename=expanded_file_name,
             level=logging.INFO,

freezer/scheduler/freezer_scheduler.py

@@ -29,7 +29,7 @@ from freezer.scheduler import arguments
 from freezer.scheduler import scheduler_job
 from freezer.scheduler import shell
 from freezer.scheduler import utils
-from freezer import winutils
+from freezer.utils import winutils
 
 
 if winutils.is_windows():

freezer/scheduler/shell.py

@@ -29,7 +29,7 @@ except Exception:
     def pprint(doc):
         print(json.dumps(doc, indent=4))
 
-from freezer.utils import create_dir
+from freezer.utils import utils as freezer_utils
 
 
 def do_session_remove_job(client, args):
@@ -174,7 +174,7 @@ def do_job_stop(client, args):
 
 
 def do_job_download(client, args):
-    create_dir(args.jobs_dir, do_log=True)
+    freezer_utils.create_dir(args.jobs_dir, do_log=True)
     for doc in _job_list(client, args):
         fname = os.path.normpath('{0}/job_{1}.conf'.
                                  format(args.jobs_dir, doc['job_id']))

freezer/scheduler/win_daemon.py

@@ -17,8 +17,8 @@ import os
 
 import win32serviceutil
 
-from freezer import utils
-from freezer import winutils
+from freezer.utils import utils
+from freezer.utils import winutils
 
 
 def setup_logging(log_file):

freezer/scheduler/win_service.py

@@ -19,7 +19,7 @@ import win32event
 import win32service
 import win32serviceutil
 
-from freezer.winutils import set_environment
+from freezer.utils import winutils
 
 
 class PySvc(win32serviceutil.ServiceFramework):
@@ -65,7 +65,7 @@ class PySvc(win32serviceutil.ServiceFramework):
             servicemanager.PYS_SERVICE_STARTED,
             (self._svc_name_, ''))
 
-        set_environment(self.home)
+        winutils.set_environment(self.home)
 
         if os.environ.get('SERVICE_INSECURE'):
             self.insecure = True

freezer/snapshot/lvm.py

@@ -23,7 +23,7 @@ import subprocess
 import uuid
 
 from freezer.common import config as freezer_config
-from freezer import utils
+from freezer.utils import utils
 
 
 def lvm_snap_remove(backup_opt_dict):
@@ -47,6 +47,36 @@ def lvm_snap_remove(backup_opt_dict):
     logging.info('[*] Snapshot volume {0} removed'.format(lv))
 
 
+def get_vol_fs_type(vol_name):
+    """
+    The argument need to be a full path lvm name i.e. /dev/vg0/var
+    or a disk partition like /dev/sda1. The returnet value is the
+    file system type
+    """
+    if os.path.exists(vol_name) is False:
+        err = '[*] Provided volume name not found: {0} '.format(vol_name)
+        logging.exception(err)
+        raise Exception(err)
+
+    file_cmd = '{0} -0 -bLs --no-pad --no-buffer --preserve-date \
+    {1}'.format(utils.find_executable("file"), vol_name)
+    file_process = subprocess.Popen(
+        file_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE, shell=True,
+        executable=utils.find_executable("bash"))
+    (file_out, file_err) = file_process.communicate()
+    file_match = re.search(r'(\S+?) filesystem data', file_out, re.I)
+    if file_match is None:
+        err = '[*] File system type not guessable: {0}'.format(file_err)
+        logging.exception(err)
+        raise Exception(err)
+    else:
+        filesys_type = file_match.group(1)
+        logging.info('[*] File system {0} found for volume {1}'.format(
+            filesys_type, vol_name))
+        return filesys_type.lower().strip()
+
+
 def lvm_snap(backup_opt_dict):
     """
     Checks the provided parameters and create the lvm snapshot if requested
@@ -141,7 +171,7 @@ def lvm_snap(backup_opt_dict):
 
     # Guess the file system of the provided source volume and st mount
     # options accordingly
-    filesys_type = utils.get_vol_fs_type(backup_opt_dict.lvm_srcvol)
+    filesys_type = get_vol_fs_type(backup_opt_dict.lvm_srcvol)
     mount_options = '-o {}'.format(backup_opt_dict.lvm_snapperm)
     if 'xfs' == filesys_type:
         mount_options = ' -onouuid '

freezer/snapshot/snapshot.py

@@ -0,0 +1,62 @@
+"""
+(c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Freezer Backup modes related functions
+"""
+import os
+
+from oslo_config import cfg
+from oslo_log import log
+
+from freezer.snapshot import lvm
+from freezer.snapshot import vss
+from freezer.utils import winutils
+
+
+CONF = cfg.CONF
+logging = log.getLogger(__name__)
+home = os.path.expanduser("~")
+
+
+def snapshot_create(backup_opt_dict):
+    """
+    Calls the code to take fs snapshots, depending on the platform
+
+    :param backup_opt_dict:
+    :return: boolean value, True if snapshot has been taken, false otherwise
+    """
+
+    if winutils.is_windows():
+        if backup_opt_dict.snapshot:
+            # Create a shadow copy.
+            backup_opt_dict.shadow_path, backup_opt_dict.shadow = \
+                vss.vss_create_shadow_copy(backup_opt_dict.windows_volume)
+
+            backup_opt_dict.path_to_backup = winutils.use_shadow(
+                backup_opt_dict.path_to_backup,
+                backup_opt_dict.windows_volume)
+            return True
+        return False
+    else:
+        return lvm.lvm_snap(backup_opt_dict)
+
+
+def snapshot_remove(backup_opt_dict, shadow, windows_volume):
+    if winutils.is_windows():
+        # Delete the shadow copy after the backup
+        vss.vss_delete_shadow_copy(shadow, windows_volume)
+    else:
+        # Unmount and remove lvm snapshot volume
+        lvm.lvm_snap_remove(backup_opt_dict)

freezer/snapshot/vss.py

@@ -15,8 +15,8 @@
 import logging
 import os
 
-from freezer import utils
-from freezer import winutils
+from freezer.utils import utils
+from freezer.utils import winutils
 
 
 def vss_create_shadow_copy(windows_volume):

freezer/storage/base.py

@@ -12,12 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from freezer import utils
-
 import logging
 import os
 import re
 
+from freezer.utils import utils
+
 
 class Storage(object):
     """

freezer/storage/fslike.py

@@ -14,7 +14,7 @@
 
 
 from freezer.storage import base
-from freezer import utils
+from freezer.utils import utils
 
 
 class FsLikeStorage(base.Storage):

freezer/storage/local.py

@@ -20,7 +20,7 @@ import os
 import shutil
 
 from freezer.storage import fslike
-from freezer import utils
+from freezer.utils import utils
 
 
 class LocalStorage(fslike.FsLikeStorage):

freezer/storage/multiple.py

@@ -12,11 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from freezer.storage import base
-from freezer import streaming
 from oslo_config import cfg
 from oslo_log import log
 
+from freezer.storage import base
+from freezer.utils import streaming
+
 CONF = cfg.CONF
 logging = log.getLogger(__name__)
 

freezer/storage/ssh.py

@@ -22,7 +22,7 @@ import paramiko
 
 from freezer.storage import fslike
 
-from freezer import utils
+from freezer.utils import utils
 
 
 class SshStorage(fslike.FsLikeStorage):

freezer/tests/commons.py

@@ -26,7 +26,8 @@ import pymongo
 import re
 from glanceclient.common.utils import IterableWithLength
 from freezer.storage import swift
-from freezer.utils import OpenstackOptions
+from freezer.utils import utils
+from freezer.openstack import openstack
 from freezer.engine.tar import tar_engine
 
 os.environ['OS_REGION_NAME'] = 'testregion'
@@ -315,8 +316,8 @@ class BackupOpt1:
         self.cindernative_vol_id = ''
         self.nova_inst_id = ''
         self.lvm_snapperm = 'ro'
-        self.options = OpenstackOptions.create_from_dict(os.environ)
-        from freezer.osclients import ClientManager
+        self.options = openstack.OpenstackOptions.create_from_dict(os.environ)
+        from freezer.openstack.osclients import ClientManager
         from mock import Mock
         self.client_manager = ClientManager(None, False, 2, False)
         self.client_manager.get_swift = Mock(

freezer/utils/config.py

@@ -12,14 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from freezer import utils
 
 import logging
 import os
 import re
+
 from six.moves import configparser
 from six.moves import cStringIO
 
+from freezer.utils import utils
+
 
 class Config:
 

freezer/utils/utils.py

@@ -19,79 +19,19 @@ import datetime
 import errno
 import logging
 import os
-import re
 import subprocess
 import sys
 import time
 
 from distutils import spawn as distspawn
 from functools import wraps
+from oslo_config import cfg
+from oslo_log import log
 from six.moves import configparser
 
 
-class OpenstackOptions:
-    """
-    Stores credentials for OpenStack API.
-    Can be created using
-    >> create_from_env()
-    or
-    >> create_from_dict(dict)
-    """
-    def __init__(self, user_name, tenant_name, project_name, auth_url,
-                 password, identity_api_version, tenant_id=None,
-                 region_name=None, endpoint_type=None, cert=None,
-                 insecure=False, verify=True):
-        self.user_name = user_name
-        self.tenant_name = tenant_name
-        self.auth_url = auth_url
-        self.password = password
-        self.tenant_id = tenant_id
-        self.project_name = project_name
-        self.identity_api_version = identity_api_version
-        self.region_name = region_name
-        self.endpoint_type = endpoint_type
-        self.cert = cert
-        self.insecure = insecure
-        self.verify = verify
-        if not (self.password and self.user_name and self.auth_url and
-           (self.tenant_name or self.project_name)):
-            raise Exception("Please set up in your env:"
-                            "OS_USERNAME, OS_TENANT_NAME/OS_PROJECT_NAME,"
-                            " OS_AUTH_URL, OS_PASSWORD")
-
-    @property
-    def os_options(self):
-        """
-        :return: The OpenStack options which can have tenant_id,
-                 auth_token, service_type, endpoint_type, tenant_name,
-                 object_storage_url, region_name
-        """
-        return {'tenant_id': self.tenant_id,
-                'tenant_name': self.tenant_name,
-                'project_name': self.project_name,
-                'identity_api_version': self.identity_api_version,
-                'region_name': self.region_name,
-                'endpoint_type': self.endpoint_type}
-
-    @staticmethod
-    def create_from_env():
-        return OpenstackOptions.create_from_dict(os.environ)
-
-    @staticmethod
-    def create_from_dict(src_dict):
-        return OpenstackOptions(
-            user_name=src_dict.get('OS_USERNAME', None),
-            tenant_name=src_dict.get('OS_TENANT_NAME', None),
-            project_name=src_dict.get('OS_PROJECT_NAME', None),
-            auth_url=src_dict.get('OS_AUTH_URL', None),
-            identity_api_version=src_dict.get('OS_IDENTITY_API_VERSION',
-                                              '2.0'),
-            password=src_dict.get('OS_PASSWORD', None),
-            tenant_id=src_dict.get('OS_TENANT_ID', None),
-            region_name=src_dict.get('OS_REGION_NAME', None),
-            endpoint_type=src_dict.get('OS_ENDPOINT_TYPE', None),
-            cert=src_dict.get('OS_CERT', None)
-        )
+CONF = cfg.CONF
+LOG = log.getLogger(__name__)
 
 
 def create_dir_tree(dir):
@@ -164,36 +104,6 @@ class DateTime(object):
         return DateTime(datetime.datetime.now())
 
 
-def get_vol_fs_type(vol_name):
-    """
-    The argument need to be a full path lvm name i.e. /dev/vg0/var
-    or a disk partition like /dev/sda1. The returnet value is the
-    file system type
-    """
-    if os.path.exists(vol_name) is False:
-        err = '[*] Provided volume name not found: {0} '.format(vol_name)
-        logging.exception(err)
-        raise Exception(err)
-
-    file_cmd = '{0} -0 -bLs --no-pad --no-buffer --preserve-date \
-    {1}'.format(find_executable("file"), vol_name)
-    file_process = subprocess.Popen(
-        file_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE, shell=True,
-        executable=find_executable("bash"))
-    (file_out, file_err) = file_process.communicate()
-    file_match = re.search(r'(\S+?) filesystem data', file_out, re.I)
-    if file_match is None:
-        err = '[*] File system type not guessable: {0}'.format(file_err)
-        logging.exception(err)
-        raise Exception(err)
-    else:
-        filesys_type = file_match.group(1)
-        logging.info('[*] File system {0} found for volume {1}'.format(
-            filesys_type, vol_name))
-        return filesys_type.lower().strip()
-
-
 def path_join(*args):
     """Should work for windows and linux"""
     return "/".join([str(x) for x in args])
@@ -361,7 +271,7 @@ def find_executable(name):
 
 
 def openssl_path():
-    from freezer import winutils
+    from freezer.utils import winutils
     if winutils.is_windows():
         return 'openssl'
     else:
@@ -370,7 +280,7 @@ def openssl_path():
 
 def tar_path():
     """This function returns tar binary path"""
-    from freezer import winutils
+    from freezer.utils import winutils
     if winutils.is_windows():
         path_to_binaries = os.path.dirname(os.path.abspath(__file__))
         return '{0}\\bin\\tar.exe'.format(path_to_binaries)
@@ -392,7 +302,7 @@ def get_executable_path(binary):
     :rtype: str
     :return: Absoulte Path to the executable file
     """
-    from freezer import winutils
+    from freezer.utils import winutils
     if winutils.is_windows():
         path_to_binaries = os.path.dirname(os.path.abspath(__file__))
         return '{0}\\bin\\{1}.exe'.format(path_to_binaries, binary)
@@ -520,3 +430,22 @@ class Namespace(dict):
     @staticmethod
     def delattr(ns, name):
         return object.__delattr__(ns, name)
+
+
+def set_max_process_priority():
+    """ Set freezer in max priority on the os """
+    # children processes inherit niceness from father
+    try:
+        LOG.warning(
+            '[*] Setting freezer execution with high CPU and I/O priority')
+        PID = os.getpid()
+        # Set cpu priority
+        os.nice(-19)
+        # Set I/O Priority to Real Time class with level 0
+        subprocess.call([
+            u'{0}'.format(find_executable("ionice")),
+            u'-c', u'1', u'-n', u'0', u'-t',
+            u'-p', u'{0}'.format(PID)
+        ])
+    except Exception as priority_error:
+        LOG.warning('[*] Priority: {0}'.format(priority_error))

freezer/utils/winutils.py

@@ -14,13 +14,10 @@
 
 import ctypes
 import json
-import logging
 import os
 import six
 import sys
 
-from freezer.utils import create_subprocess
-
 
 def is_windows():
     """

tests/unit/engines/tar/test_tar_builders.py

@@ -14,8 +14,9 @@
 
 
 import unittest
+
 from freezer.engine.tar import tar_builders
-from freezer import utils
+from freezer.utils import utils
 
 
 class TestTarCommandBuilder(unittest.TestCase):

tests/unit/openstack/test_osclients.py

@@ -16,13 +16,14 @@
 import unittest
 import mock
 
-from freezer import osclients
-from freezer import utils
+from freezer.openstack import openstack
+from freezer.openstack import osclients
+from freezer.utils import utils
 
 
 class TestOsClients(unittest.TestCase):
 
-    fake_options = utils.OpenstackOptions(
+    fake_options = openstack.OpenstackOptions(
         user_name="user", tenant_name="tenant", project_name="project",
         auth_url="url", password="password", identity_api_version="3",
         insecure=False, cert='cert', verify=True)
@@ -43,7 +44,7 @@ class TestOsClients(unittest.TestCase):
         client.create_nova()
 
     def test_create_swift_public(self):
-        options = utils.OpenstackOptions(
+        options = openstack.OpenstackOptions(
             user_name="user", tenant_name="tenant", project_name="project",
             auth_url="url", password="password", identity_api_version="3",
             endpoint_type="adminURL", insecure=False, cert='cert',
@@ -68,4 +69,4 @@ class TestOsClients(unittest.TestCase):
 
     def get_nova(self):
         client = osclients.ClientManager(self.fake_options, None, None, None)
-        client.get_nova()
+        client.get_nova()

tests/unit/openstack/test_restore.py

@@ -17,7 +17,7 @@ limitations under the License.
 """
 import unittest
 
-from freezer import restore
+from freezer.openstack import restore
 from freezer.tests import commons
 
 

tests/unit/scheduler/test_scheduler_daemon.py

@@ -28,7 +28,7 @@ class TestLogging(unittest.TestCase):
         res = daemon.setup_logging(None)
         self.assertEqual('/var/log/freezer-scheduler.log', res)
 
-    @patch('