diff --git a/freezer/arguments.py b/freezer/arguments.py index c42d54f1..6f5d7601 100644 --- a/freezer/arguments.py +++ b/freezer/arguments.py @@ -36,6 +36,7 @@ import utils from oslo_utils import encodeutils from freezer import winutils +from tempfile import NamedTemporaryFile home = expanduser("~") @@ -47,7 +48,7 @@ DEFAULT_PARAMS = { 'backup_name': False, 'quiet': False, 'container': 'freezer_backups', 'no_incremental': False, 'max_segment_size': 67108864, 'lvm_srcvol': False, - 'download_limit': -1, 'hostname': False, 'remove_from_date': False, + 'download_limit': None, 'hostname': False, 'remove_from_date': False, 'restart_always_level': False, 'lvm_dirmount': False, 'dst_file': False, 'dereference_symlink': '', 'restore_from_host': False, 'config': False, 'mysql_conf': False, @@ -57,7 +58,7 @@ DEFAULT_PARAMS = { 'cinder_vol_id': '', 'cindernative_vol_id': '', 'nova_inst_id': '', 'list_containers': False, 'remove_older_than': None, 'restore_from_date': False, - 'upload_limit': -1, 'always_level': False, 'version': False, + 'upload_limit': None, 'always_level': False, 'version': False, 'dry_run': False, 'lvm_snapsize': False, 'restore_abs_path': False, 'log_file': None, 'upload': True, 'mode': 'fs', 'action': 'backup', @@ -114,6 +115,11 @@ def backup_arguments(args_dict={}): defaults = DEFAULT_PARAMS.copy() args, remaining_argv = conf_parser.parse_known_args() if args.config: + if not os.path.exists(args.config): + logging.error("[*] Critical Error: Configuration file {0} not" + " found".format(args.config)) + raise Exception("Configuration file {0} not found !".format( + args.config)) config = configparser.SafeConfigParser() config.read([args.config]) section = config.sections()[0] @@ -561,4 +567,58 @@ def backup_arguments(args_dict={}): backup_args.__dict__['time_stamp'] = None + if backup_args.upload_limit or backup_args.download_limit and not\ + winutils.is_windows(): + if backup_args.config: + conf_file = NamedTemporaryFile(prefix='freezer_job_', delete=False) + defaults['upload_limit'] = defaults['download_limit'] = -1 + utils.save_config_to_file(defaults, conf_file) + conf_index = sys.argv.index('--config') + 1 + sys.argv[conf_index] = conf_file.name + + if '--upload-limit' in sys.argv: + index = sys.argv.index('--upload-limit') + sys.argv.pop(index) + sys.argv.pop(index) + if '--download-limit' in sys.argv: + index = sys.argv.index('--download-limit') + sys.argv.pop(index) + sys.argv.pop(index) + + trickle_executable = distspawn.find_executable('trickle') + if trickle_executable is None: + trickle_executable = distspawn.find_executable( + 'trickle', path=":".join(sys.path)) + if trickle_executable is None: + trickle_executable = distspawn.find_executable( + 'trickle', path=":".join(os.environ.get('PATH'))) + + trickle_lib = distspawn.find_executable('trickle-overload.so') + if trickle_lib is None: + trickle_lib = distspawn.find_executable( + 'trickle-overload.so', path=":".join(sys.path)) + if trickle_lib is None: + trickle_lib = distspawn.find_executable( + 'trickle-overload.so', path=":".join( + os.environ.get('PATH'))) + if trickle_executable and trickle_lib: + logging.info("[*] Info: Starting trickle ...") + os.environ['LD_PRELOAD'] = trickle_lib + trickle_command = '{0} -d {1} -u {2} '.\ + format(trickle_executable, + getattr(backup_args, 'download_limit') or -1, + getattr(backup_args, 'upload_limit') or -1) + backup_args.__dict__['trickle_command'] = trickle_command + if "tricklecount" in os.environ: + tricklecount = int(os.environ.get("tricklecount", 1)) + tricklecount += 1 + os.environ["tricklecount"] = str(tricklecount) + + else: + os.environ["tricklecount"] = str(1) + else: + logging.critical("[*] Trickle or Trickle library not found" + ". Switching to normal mode without limiting" + " bandwidth") + return backup_args, arg_parser diff --git a/freezer/bin/trickle b/freezer/bin/trickle new file mode 100755 index 00000000..a0c332d4 Binary files /dev/null and b/freezer/bin/trickle differ diff --git a/freezer/bin/trickle-overload.so b/freezer/bin/trickle-overload.so new file mode 100755 index 00000000..72eb7b93 Binary files /dev/null and b/freezer/bin/trickle-overload.so differ diff --git a/freezer/bin/trickle.tgz b/freezer/bin/trickle.tgz new file mode 100644 index 00000000..1c3c82fc Binary files /dev/null and b/freezer/bin/trickle.tgz differ diff --git a/freezer/main.py b/freezer/main.py index 9f7cf349..f65bda1e 100644 --- a/freezer/main.py +++ b/freezer/main.py @@ -155,8 +155,33 @@ def freezer_main(backup_args, arg_parse): backup_args.encrypt_pass_file, backup_args.dry_run) - freezer_job = job.create_job(backup_args) - freezer_job.execute() + if hasattr(backup_args, 'trickle_command'): + if "tricklecount" in os.environ: + if int(os.environ.get("tricklecount")) > 1: + logging.critical("[*] Trickle seems to be not working," + " Switching to normal mode ") + freezer_job = job.create_job(backup_args) + freezer_job.execute() + + freezer_command = '{0} {1}'.format(backup_args.trickle_command, + ' '.join(sys.argv)) + process = subprocess.Popen(freezer_command.split(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=os.environ.copy()) + while process.poll() is None: + print process.stdout.readline().rstrip() + output, error = process.communicate() + + if process.returncode: + logging.error("[*] Trickle Error: {0}".format(error)) + logging.critical("[*] Switching to work without trickle ...") + freezer_job = job.create_job(backup_args) + freezer_job.execute() + + else: + freezer_job = job.create_job(backup_args) + freezer_job.execute() if backup_args.metadata_out == '-': metadata = freezer_job.get_metadata() diff --git a/freezer/utils.py b/freezer/utils.py index 86b70d54..3d4623cf 100644 --- a/freezer/utils.py +++ b/freezer/utils.py @@ -28,6 +28,7 @@ import datetime import re import subprocess import errno +from ConfigParser import ConfigParser class OpenstackOptions: @@ -125,6 +126,15 @@ def create_dir(directory, do_log=True): raise Exception(err) +def save_config_to_file(config, f, section='freezer_default'): + parser = ConfigParser() + parser.add_section(section) + for option, option_value in config.items(): + parser.set(section, option, option_value) + parser.write(f) + f.close() + + class DateTime(object): def __init__(self, value): if isinstance(value, int): diff --git a/setup.py b/setup.py index 77865b79..2c314028 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,9 @@ setup( packages=find_packages(), platforms='Linux, *BSD, OSX', cmdclass={'test': PyTest}, - scripts=['bin/freezerc'], + scripts=['bin/freezerc', 'freezer/bin/trickle', + 'freezer/bin/trickle-overload.so'], + include_package_data=True, classifiers=[ 'Programming Language :: Python', 'Development Status :: 5 - Production/Stable', @@ -119,5 +121,7 @@ setup( ('freezer/bin', ['freezer/bin/gzip-1.6-1.src.tar']), ('freezer/bin', ['freezer/bin/tar-1.27.1.tar.xz']), ('freezer/bin', ['freezer/bin/findutils-4.5.12.tar.gz']), - ('freezer/bin', ['freezer/bin/xz-5.2.1.tar.gz'])] + ('freezer/bin', ['freezer/bin/xz-5.2.1.tar.gz']), + ('freezer/bin', ['freezer/bin/trickle']), + ('freezer/bin', ['freezer/bin/trickle-overload.so'])] )