Throttling bandwidth on Linux for HTTPS

using trickle to limit bandwidth on linux by allowing freezer agent
to run a fork of itself through trickle and passing limit parameters
(upload-limit, download-limit) to trickle and remove it from
freezer agent.

Closes-Bug: #1466858

Change-Id: I0c3ac2e4085389dc71c8e242c84fefe7aea3c8a5
This commit is contained in:
Saad Zaher 2015-09-23 18:22:19 +01:00
parent d7f00faf51
commit 959f042726
7 changed files with 105 additions and 6 deletions

View File

@ -36,6 +36,7 @@ import utils
from oslo_utils import encodeutils
from freezer import winutils
from tempfile import NamedTemporaryFile
home = expanduser("~")
@ -47,7 +48,7 @@ DEFAULT_PARAMS = {
'backup_name': False, 'quiet': False,
'container': 'freezer_backups', 'no_incremental': False,
'max_segment_size': 67108864, 'lvm_srcvol': False,
'download_limit': -1, 'hostname': False, 'remove_from_date': False,
'download_limit': None, 'hostname': False, 'remove_from_date': False,
'restart_always_level': False, 'lvm_dirmount': False,
'dst_file': False, 'dereference_symlink': '',
'restore_from_host': False, 'config': False, 'mysql_conf': False,
@ -57,7 +58,7 @@ DEFAULT_PARAMS = {
'cinder_vol_id': '', 'cindernative_vol_id': '',
'nova_inst_id': '', 'list_containers': False,
'remove_older_than': None, 'restore_from_date': False,
'upload_limit': -1, 'always_level': False, 'version': False,
'upload_limit': None, 'always_level': False, 'version': False,
'dry_run': False, 'lvm_snapsize': False,
'restore_abs_path': False, 'log_file': None,
'upload': True, 'mode': 'fs', 'action': 'backup',
@ -114,6 +115,11 @@ def backup_arguments(args_dict={}):
defaults = DEFAULT_PARAMS.copy()
args, remaining_argv = conf_parser.parse_known_args()
if args.config:
if not os.path.exists(args.config):
logging.error("[*] Critical Error: Configuration file {0} not"
" found".format(args.config))
raise Exception("Configuration file {0} not found !".format(
args.config))
config = configparser.SafeConfigParser()
config.read([args.config])
section = config.sections()[0]
@ -561,4 +567,58 @@ def backup_arguments(args_dict={}):
backup_args.__dict__['time_stamp'] = None
if backup_args.upload_limit or backup_args.download_limit and not\
winutils.is_windows():
if backup_args.config:
conf_file = NamedTemporaryFile(prefix='freezer_job_', delete=False)
defaults['upload_limit'] = defaults['download_limit'] = -1
utils.save_config_to_file(defaults, conf_file)
conf_index = sys.argv.index('--config') + 1
sys.argv[conf_index] = conf_file.name
if '--upload-limit' in sys.argv:
index = sys.argv.index('--upload-limit')
sys.argv.pop(index)
sys.argv.pop(index)
if '--download-limit' in sys.argv:
index = sys.argv.index('--download-limit')
sys.argv.pop(index)
sys.argv.pop(index)
trickle_executable = distspawn.find_executable('trickle')
if trickle_executable is None:
trickle_executable = distspawn.find_executable(
'trickle', path=":".join(sys.path))
if trickle_executable is None:
trickle_executable = distspawn.find_executable(
'trickle', path=":".join(os.environ.get('PATH')))
trickle_lib = distspawn.find_executable('trickle-overload.so')
if trickle_lib is None:
trickle_lib = distspawn.find_executable(
'trickle-overload.so', path=":".join(sys.path))
if trickle_lib is None:
trickle_lib = distspawn.find_executable(
'trickle-overload.so', path=":".join(
os.environ.get('PATH')))
if trickle_executable and trickle_lib:
logging.info("[*] Info: Starting trickle ...")
os.environ['LD_PRELOAD'] = trickle_lib
trickle_command = '{0} -d {1} -u {2} '.\
format(trickle_executable,
getattr(backup_args, 'download_limit') or -1,
getattr(backup_args, 'upload_limit') or -1)
backup_args.__dict__['trickle_command'] = trickle_command
if "tricklecount" in os.environ:
tricklecount = int(os.environ.get("tricklecount", 1))
tricklecount += 1
os.environ["tricklecount"] = str(tricklecount)
else:
os.environ["tricklecount"] = str(1)
else:
logging.critical("[*] Trickle or Trickle library not found"
". Switching to normal mode without limiting"
" bandwidth")
return backup_args, arg_parser

BIN
freezer/bin/trickle Executable file

Binary file not shown.

BIN
freezer/bin/trickle-overload.so Executable file

Binary file not shown.

BIN
freezer/bin/trickle.tgz Normal file

Binary file not shown.

View File

@ -155,8 +155,33 @@ def freezer_main(backup_args, arg_parse):
backup_args.encrypt_pass_file,
backup_args.dry_run)
freezer_job = job.create_job(backup_args)
freezer_job.execute()
if hasattr(backup_args, 'trickle_command'):
if "tricklecount" in os.environ:
if int(os.environ.get("tricklecount")) > 1:
logging.critical("[*] Trickle seems to be not working,"
" Switching to normal mode ")
freezer_job = job.create_job(backup_args)
freezer_job.execute()
freezer_command = '{0} {1}'.format(backup_args.trickle_command,
' '.join(sys.argv))
process = subprocess.Popen(freezer_command.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=os.environ.copy())
while process.poll() is None:
print process.stdout.readline().rstrip()
output, error = process.communicate()
if process.returncode:
logging.error("[*] Trickle Error: {0}".format(error))
logging.critical("[*] Switching to work without trickle ...")
freezer_job = job.create_job(backup_args)
freezer_job.execute()
else:
freezer_job = job.create_job(backup_args)
freezer_job.execute()
if backup_args.metadata_out == '-':
metadata = freezer_job.get_metadata()

View File

@ -28,6 +28,7 @@ import datetime
import re
import subprocess
import errno
from ConfigParser import ConfigParser
class OpenstackOptions:
@ -125,6 +126,15 @@ def create_dir(directory, do_log=True):
raise Exception(err)
def save_config_to_file(config, f, section='freezer_default'):
parser = ConfigParser()
parser.add_section(section)
for option, option_value in config.items():
parser.set(section, option, option_value)
parser.write(f)
f.close()
class DateTime(object):
def __init__(self, value):
if isinstance(value, int):

View File

@ -56,7 +56,9 @@ setup(
packages=find_packages(),
platforms='Linux, *BSD, OSX',
cmdclass={'test': PyTest},
scripts=['bin/freezerc'],
scripts=['bin/freezerc', 'freezer/bin/trickle',
'freezer/bin/trickle-overload.so'],
include_package_data=True,
classifiers=[
'Programming Language :: Python',
'Development Status :: 5 - Production/Stable',
@ -119,5 +121,7 @@ setup(
('freezer/bin', ['freezer/bin/gzip-1.6-1.src.tar']),
('freezer/bin', ['freezer/bin/tar-1.27.1.tar.xz']),
('freezer/bin', ['freezer/bin/findutils-4.5.12.tar.gz']),
('freezer/bin', ['freezer/bin/xz-5.2.1.tar.gz'])]
('freezer/bin', ['freezer/bin/xz-5.2.1.tar.gz']),
('freezer/bin', ['freezer/bin/trickle']),
('freezer/bin', ['freezer/bin/trickle-overload.so'])]
)