Throttling bandwidth on Linux for HTTPS
using trickle to limit bandwidth on linux by allowing freezer agent to run a fork of itself through trickle and passing limit parameters (upload-limit, download-limit) to trickle and remove it from freezer agent. Closes-Bug: #1466858 Change-Id: I0c3ac2e4085389dc71c8e242c84fefe7aea3c8a5
This commit is contained in:
Saad Zaher
parent
d7f00faf51
commit
959f042726
@ -36,6 +36,7 @@ import utils
|
||||
from oslo_utils import encodeutils
|
||||
|
||||
from freezer import winutils
|
||||
from tempfile import NamedTemporaryFile
|
||||
|
||||
home = expanduser("~")
|
||||
|
||||
@ -47,7 +48,7 @@ DEFAULT_PARAMS = {
|
||||
'backup_name': False, 'quiet': False,
|
||||
'container': 'freezer_backups', 'no_incremental': False,
|
||||
'max_segment_size': 67108864, 'lvm_srcvol': False,
|
||||
'download_limit': -1, 'hostname': False, 'remove_from_date': False,
|
||||
'download_limit': None, 'hostname': False, 'remove_from_date': False,
|
||||
'restart_always_level': False, 'lvm_dirmount': False,
|
||||
'dst_file': False, 'dereference_symlink': '',
|
||||
'restore_from_host': False, 'config': False, 'mysql_conf': False,
|
||||
@ -57,7 +58,7 @@ DEFAULT_PARAMS = {
|
||||
'cinder_vol_id': '', 'cindernative_vol_id': '',
|
||||
'nova_inst_id': '', 'list_containers': False,
|
||||
'remove_older_than': None, 'restore_from_date': False,
|
||||
'upload_limit': -1, 'always_level': False, 'version': False,
|
||||
'upload_limit': None, 'always_level': False, 'version': False,
|
||||
'dry_run': False, 'lvm_snapsize': False,
|
||||
'restore_abs_path': False, 'log_file': None,
|
||||
'upload': True, 'mode': 'fs', 'action': 'backup',
|
||||
@ -114,6 +115,11 @@ def backup_arguments(args_dict={}):
|
||||
defaults = DEFAULT_PARAMS.copy()
|
||||
args, remaining_argv = conf_parser.parse_known_args()
|
||||
if args.config:
|
||||
if not os.path.exists(args.config):
|
||||
logging.error("[*] Critical Error: Configuration file {0} not"
|
||||
" found".format(args.config))
|
||||
raise Exception("Configuration file {0} not found !".format(
|
||||
args.config))
|
||||
config = configparser.SafeConfigParser()
|
||||
config.read([args.config])
|
||||
section = config.sections()[0]
|
||||
@ -561,4 +567,58 @@ def backup_arguments(args_dict={}):
|
||||
|
||||
backup_args.__dict__['time_stamp'] = None
|
||||
|
||||
if backup_args.upload_limit or backup_args.download_limit and not\
|
||||
winutils.is_windows():
|
||||
if backup_args.config:
|
||||
conf_file = NamedTemporaryFile(prefix='freezer_job_', delete=False)
|
||||
defaults['upload_limit'] = defaults['download_limit'] = -1
|
||||
utils.save_config_to_file(defaults, conf_file)
|
||||
conf_index = sys.argv.index('--config') + 1
|
||||
sys.argv[conf_index] = conf_file.name
|
||||
|
||||
if '--upload-limit' in sys.argv:
|
||||
index = sys.argv.index('--upload-limit')
|
||||
sys.argv.pop(index)
|
||||
sys.argv.pop(index)
|
||||
if '--download-limit' in sys.argv:
|
||||
index = sys.argv.index('--download-limit')
|
||||
sys.argv.pop(index)
|
||||
sys.argv.pop(index)
|
||||
|
||||
trickle_executable = distspawn.find_executable('trickle')
|
||||
if trickle_executable is None:
|
||||
trickle_executable = distspawn.find_executable(
|
||||
'trickle', path=":".join(sys.path))
|
||||
if trickle_executable is None:
|
||||
trickle_executable = distspawn.find_executable(
|
||||
'trickle', path=":".join(os.environ.get('PATH')))
|
||||
|
||||
trickle_lib = distspawn.find_executable('trickle-overload.so')
|
||||
if trickle_lib is None:
|
||||
trickle_lib = distspawn.find_executable(
|
||||
'trickle-overload.so', path=":".join(sys.path))
|
||||
if trickle_lib is None:
|
||||
trickle_lib = distspawn.find_executable(
|
||||
'trickle-overload.so', path=":".join(
|
||||
os.environ.get('PATH')))
|
||||
if trickle_executable and trickle_lib:
|
||||
logging.info("[*] Info: Starting trickle ...")
|
||||
os.environ['LD_PRELOAD'] = trickle_lib
|
||||
trickle_command = '{0} -d {1} -u {2} '.\
|
||||
format(trickle_executable,
|
||||
getattr(backup_args, 'download_limit') or -1,
|
||||
getattr(backup_args, 'upload_limit') or -1)
|
||||
backup_args.__dict__['trickle_command'] = trickle_command
|
||||
if "tricklecount" in os.environ:
|
||||
tricklecount = int(os.environ.get("tricklecount", 1))
|
||||
tricklecount += 1
|
||||
os.environ["tricklecount"] = str(tricklecount)
|
||||
|
||||
else:
|
||||
os.environ["tricklecount"] = str(1)
|
||||
else:
|
||||
logging.critical("[*] Trickle or Trickle library not found"
|
||||
". Switching to normal mode without limiting"
|
||||
" bandwidth")
|
||||
|
||||
return backup_args, arg_parser
|
||||
|
||||
BIN
freezer/bin/trickle
Executable file
BIN
freezer/bin/trickle
Executable file
Binary file not shown.
BIN
freezer/bin/trickle-overload.so
Executable file
BIN
freezer/bin/trickle-overload.so
Executable file
Binary file not shown.
BIN
freezer/bin/trickle.tgz
Normal file
BIN
freezer/bin/trickle.tgz
Normal file
Binary file not shown.
@ -155,8 +155,33 @@ def freezer_main(backup_args, arg_parse):
|
||||
backup_args.encrypt_pass_file,
|
||||
backup_args.dry_run)
|
||||
|
||||
freezer_job = job.create_job(backup_args)
|
||||
freezer_job.execute()
|
||||
if hasattr(backup_args, 'trickle_command'):
|
||||
if "tricklecount" in os.environ:
|
||||
if int(os.environ.get("tricklecount")) > 1:
|
||||
logging.critical("[*] Trickle seems to be not working,"
|
||||
" Switching to normal mode ")
|
||||
freezer_job = job.create_job(backup_args)
|
||||
freezer_job.execute()
|
||||
|
||||
freezer_command = '{0} {1}'.format(backup_args.trickle_command,
|
||||
' '.join(sys.argv))
|
||||
process = subprocess.Popen(freezer_command.split(),
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=os.environ.copy())
|
||||
while process.poll() is None:
|
||||
print process.stdout.readline().rstrip()
|
||||
output, error = process.communicate()
|
||||
|
||||
if process.returncode:
|
||||
logging.error("[*] Trickle Error: {0}".format(error))
|
||||
logging.critical("[*] Switching to work without trickle ...")
|
||||
freezer_job = job.create_job(backup_args)
|
||||
freezer_job.execute()
|
||||
|
||||
else:
|
||||
freezer_job = job.create_job(backup_args)
|
||||
freezer_job.execute()
|
||||
|
||||
if backup_args.metadata_out == '-':
|
||||
metadata = freezer_job.get_metadata()
|
||||
|
||||
@ -28,6 +28,7 @@ import datetime
|
||||
import re
|
||||
import subprocess
|
||||
import errno
|
||||
from ConfigParser import ConfigParser
|
||||
|
||||
|
||||
class OpenstackOptions:
|
||||
@ -125,6 +126,15 @@ def create_dir(directory, do_log=True):
|
||||
raise Exception(err)
|
||||
|
||||
|
||||
def save_config_to_file(config, f, section='freezer_default'):
|
||||
parser = ConfigParser()
|
||||
parser.add_section(section)
|
||||
for option, option_value in config.items():
|
||||
parser.set(section, option, option_value)
|
||||
parser.write(f)
|
||||
f.close()
|
||||
|
||||
|
||||
class DateTime(object):
|
||||
def __init__(self, value):
|
||||
if isinstance(value, int):
|
||||
|
||||
8
setup.py
8
setup.py
@ -56,7 +56,9 @@ setup(
|
||||
packages=find_packages(),
|
||||
platforms='Linux, *BSD, OSX',
|
||||
cmdclass={'test': PyTest},
|
||||
scripts=['bin/freezerc'],
|
||||
scripts=['bin/freezerc', 'freezer/bin/trickle',
|
||||
'freezer/bin/trickle-overload.so'],
|
||||
include_package_data=True,
|
||||
classifiers=[
|
||||
'Programming Language :: Python',
|
||||
'Development Status :: 5 - Production/Stable',
|
||||
@ -119,5 +121,7 @@ setup(
|
||||
('freezer/bin', ['freezer/bin/gzip-1.6-1.src.tar']),
|
||||
('freezer/bin', ['freezer/bin/tar-1.27.1.tar.xz']),
|
||||
('freezer/bin', ['freezer/bin/findutils-4.5.12.tar.gz']),
|
||||
('freezer/bin', ['freezer/bin/xz-5.2.1.tar.gz'])]
|
||||
('freezer/bin', ['freezer/bin/xz-5.2.1.tar.gz']),
|
||||
('freezer/bin', ['freezer/bin/trickle']),
|
||||
('freezer/bin', ['freezer/bin/trickle-overload.so'])]
|
||||
)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user