248 lines
8.3 KiB
Python
248 lines
8.3 KiB
Python
"""
|
|
(c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
|
|
Freezer main execution function
|
|
"""
|
|
import json
|
|
import logging
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
from freezer import arguments
|
|
from freezer import bandwidth
|
|
from freezer import config
|
|
from freezer.engine.tar import tar_engine
|
|
from freezer import job
|
|
from freezer import osclients
|
|
from freezer.storage import local
|
|
from freezer.storage import multiple
|
|
from freezer.storage import ssh
|
|
from freezer.storage import swift
|
|
from freezer import utils
|
|
from freezer import validator
|
|
from freezer import winutils
|
|
|
|
|
|
def freezer_main(backup_args):
|
|
"""Freezer main loop for job execution.
|
|
"""
|
|
|
|
def configure_log_file_using_defaults():
|
|
""" Configure log file for freezer """
|
|
|
|
dry_run_message = ''
|
|
if backup_args.dry_run:
|
|
dry_run_message = '[DRY_RUN] '
|
|
|
|
def configure_logging(log_file, str_level):
|
|
levels = {
|
|
'all': logging.NOTSET,
|
|
'debug': logging.DEBUG,
|
|
'warn': logging.WARN,
|
|
'info': logging.INFO,
|
|
'error': logging.ERROR,
|
|
'critical': logging.CRITICAL
|
|
}
|
|
|
|
expanded_file_name = os.path.expanduser(log_file)
|
|
expanded_dir_name = os.path.dirname(expanded_file_name)
|
|
utils.create_dir(expanded_dir_name, do_log=False)
|
|
logging.basicConfig(
|
|
filename=expanded_file_name,
|
|
level=levels[str_level],
|
|
format=('%(asctime)s %(name)s %(levelname)s {0}%(message)s'.
|
|
format(dry_run_message)))
|
|
return expanded_file_name
|
|
|
|
if backup_args.log_file:
|
|
return configure_logging(backup_args.log_file,
|
|
backup_args.log_level)
|
|
|
|
for file_name in ['/var/log/freezer.log', '~/.freezer/freezer.log']:
|
|
try:
|
|
return configure_logging(file_name, backup_args.log_level)
|
|
except IOError:
|
|
pass
|
|
|
|
raise Exception("Unable to write to log file")
|
|
|
|
def set_max_process_priority():
|
|
""" Set freezer in max priority on the os """
|
|
# children processes inherit niceness from father
|
|
try:
|
|
logging.warning(
|
|
'[*] Setting freezer execution with high CPU and I/O priority')
|
|
PID = os.getpid()
|
|
# Set cpu priority
|
|
os.nice(-19)
|
|
# Set I/O Priority to Real Time class with level 0
|
|
subprocess.call([
|
|
u'{0}'.format(utils.find_executable("ionice")),
|
|
u'-c', u'1', u'-n', u'0', u'-t',
|
|
u'-p', u'{0}'.format(PID)
|
|
])
|
|
except Exception as priority_error:
|
|
logging.warning('[*] Priority: {0}'.format(priority_error))
|
|
|
|
try:
|
|
log_file_name = configure_log_file_using_defaults()
|
|
except Exception as err:
|
|
fail(1, err, quiet=backup_args.quiet, do_log=False)
|
|
|
|
if not backup_args.quiet:
|
|
logging.info('log file at {0}'.format(log_file_name))
|
|
|
|
if backup_args.max_priority:
|
|
set_max_process_priority()
|
|
|
|
bandwidth.monkeypatch_socket_bandwidth(backup_args)
|
|
|
|
backup_args.__dict__['hostname_backup_name'] = "{0}_{1}".format(
|
|
backup_args.hostname, backup_args.backup_name)
|
|
|
|
validator.validate(backup_args)
|
|
|
|
work_dir = backup_args.work_dir
|
|
os_identity = backup_args.os_identity_api_version
|
|
max_segment_size = backup_args.max_segment_size
|
|
if backup_args.storages:
|
|
storage = multiple.MultipleStorage(
|
|
work_dir,
|
|
[storage_from_dict(x, work_dir, max_segment_size, os_identity)
|
|
for x in backup_args.storages])
|
|
else:
|
|
storage = storage_from_dict(backup_args.__dict__, work_dir,
|
|
max_segment_size, os_identity)
|
|
|
|
backup_args.__dict__['engine'] = tar_engine.TarBackupEngine(
|
|
backup_args.compression,
|
|
backup_args.dereference_symlink,
|
|
backup_args.exclude,
|
|
storage,
|
|
winutils.is_windows(),
|
|
backup_args.encrypt_pass_file,
|
|
backup_args.dry_run)
|
|
|
|
if hasattr(backup_args, 'trickle_command'):
|
|
if "tricklecount" in os.environ:
|
|
if int(os.environ.get("tricklecount")) > 1:
|
|
logging.critical("[*] Trickle seems to be not working,"
|
|
" Switching to normal mode ")
|
|
run_job(backup_args, storage)
|
|
|
|
freezer_command = '{0} {1}'.format(backup_args.trickle_command,
|
|
' '.join(sys.argv))
|
|
process = subprocess.Popen(freezer_command.split(),
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
env=os.environ.copy())
|
|
while process.poll() is None:
|
|
print(process.stdout.readline().rstrip())
|
|
output, error = process.communicate()
|
|
|
|
if process.returncode:
|
|
logging.error("[*] Trickle Error: {0}".format(error))
|
|
logging.critical("[*] Switching to work without trickle ...")
|
|
run_job(backup_args, storage)
|
|
|
|
else:
|
|
run_job(backup_args, storage)
|
|
|
|
|
|
def run_job(backup_args, storage):
|
|
freezer_job = job.create_job(backup_args, storage)
|
|
freezer_job.execute()
|
|
|
|
if backup_args.metadata_out == '-':
|
|
metadata = freezer_job.get_metadata()
|
|
if metadata:
|
|
sys.stdout.write(json.dumps(metadata))
|
|
|
|
|
|
def fail(exit_code, e, quiet, do_log=True):
|
|
""" Catch the exceptions and write it to log """
|
|
msg = '[*] Critical Error: {0}\n'.format(e)
|
|
if not quiet:
|
|
sys.stderr.write(msg)
|
|
if do_log:
|
|
logging.critical(msg)
|
|
return exit_code
|
|
|
|
|
|
def parse_osrc(file_name):
|
|
with open(file_name, 'r') as osrc_file:
|
|
return config.osrc_parse(osrc_file.read())
|
|
|
|
|
|
def storage_from_dict(backup_args, work_dir, max_segment_size,
|
|
os_identity_api_version=None):
|
|
storage_name = backup_args['storage']
|
|
container = backup_args['container']
|
|
if storage_name == "swift":
|
|
if "osrc" in backup_args:
|
|
options = utils.OpenstackOptions.create_from_dict(
|
|
parse_osrc(backup_args['osrc']))
|
|
else:
|
|
options = utils.OpenstackOptions.create_from_env()
|
|
identity_api_version = (os_identity_api_version or
|
|
options.identity_api_version)
|
|
client_manager = osclients.ClientManager(
|
|
options=options,
|
|
insecure=backup_args.get('insecure') or False,
|
|
swift_auth_version=identity_api_version,
|
|
dry_run=backup_args.get('dry_run') or False)
|
|
|
|
storage = swift.SwiftStorage(
|
|
client_manager, container, work_dir,max_segment_size)
|
|
backup_args['client_manager'] = client_manager
|
|
elif storage_name == "local":
|
|
storage = local.LocalStorage(container, work_dir)
|
|
elif storage_name == "ssh":
|
|
storage = ssh.SshStorage(
|
|
container, work_dir,
|
|
backup_args['ssh_key'], backup_args['ssh_username'],
|
|
backup_args['ssh_host'], int(backup_args.get('ssh_port', 22)))
|
|
else:
|
|
raise Exception("Not storage found for name " + backup_args['storage'])
|
|
return storage
|
|
|
|
|
|
def main():
|
|
"""Freezerc binary main execution"""
|
|
|
|
(backup_args, opt_args) = arguments.backup_arguments()
|
|
try:
|
|
if backup_args.version:
|
|
print("freezer version {0}".format(backup_args.__version__))
|
|
sys.exit(1)
|
|
|
|
if len(sys.argv) < 2:
|
|
opt_args.print_help()
|
|
sys.exit(1)
|
|
|
|
freezer_main(backup_args)
|
|
|
|
except ValueError as err:
|
|
return fail(1, err, backup_args.quiet)
|
|
except ImportError as err:
|
|
return fail(1, err, backup_args.quiet)
|
|
except Exception as err:
|
|
return fail(1, err, backup_args.quiet)
|
|
|
|
if __name__ == '__main__':
|
|
|
|
sys.exit(main())
|