518 lines
16 KiB
Python
518 lines
16 KiB
Python
"""
|
|
(c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
|
|
Freezer general utils functions
|
|
"""
|
|
import datetime
|
|
import errno
|
|
import logging
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
from distutils import spawn as distspawn
|
|
from functools import wraps
|
|
from six.moves import configparser
|
|
|
|
|
|
class OpenstackOptions:
|
|
"""
|
|
Stores credentials for OpenStack API.
|
|
Can be created using
|
|
>> create_from_env()
|
|
or
|
|
>> create_from_dict(dict)
|
|
"""
|
|
def __init__(self, user_name, tenant_name, project_name, auth_url,
|
|
password, identity_api_version, tenant_id=None,
|
|
region_name=None, endpoint_type=None):
|
|
self.user_name = user_name
|
|
self.tenant_name = tenant_name
|
|
self.auth_url = auth_url
|
|
self.password = password
|
|
self.tenant_id = tenant_id
|
|
self.project_name = project_name
|
|
self.identity_api_version = identity_api_version
|
|
self.region_name = region_name
|
|
self.endpoint_type = endpoint_type
|
|
if not (self.password and self.user_name and self.auth_url and
|
|
(self.tenant_name or self.project_name)):
|
|
raise Exception("Please set up in your env:"
|
|
"OS_USERNAME, OS_TENANT_NAME/OS_PROJECT_NAME,"
|
|
" OS_AUTH_URL, OS_PASSWORD")
|
|
|
|
@property
|
|
def os_options(self):
|
|
"""
|
|
:return: The OpenStack options which can have tenant_id,
|
|
auth_token, service_type, endpoint_type, tenant_name,
|
|
object_storage_url, region_name
|
|
"""
|
|
return {'tenant_id': self.tenant_id,
|
|
'tenant_name': self.tenant_name,
|
|
'project_name': self.project_name,
|
|
'identity_api_version': self.identity_api_version,
|
|
'region_name': self.region_name,
|
|
'endpoint_type': self.endpoint_type}
|
|
|
|
@staticmethod
|
|
def create_from_env():
|
|
return OpenstackOptions.create_from_dict(os.environ)
|
|
|
|
@staticmethod
|
|
def create_from_dict(src_dict):
|
|
return OpenstackOptions(
|
|
user_name=src_dict.get('OS_USERNAME', None),
|
|
tenant_name=src_dict.get('OS_TENANT_NAME', None),
|
|
project_name=src_dict.get('OS_PROJECT_NAME', None),
|
|
auth_url=src_dict.get('OS_AUTH_URL', None),
|
|
identity_api_version=src_dict.get('OS_IDENTITY_API_VERSION',
|
|
'2.0'),
|
|
password=src_dict.get('OS_PASSWORD', None),
|
|
tenant_id=src_dict.get('OS_TENANT_ID', None),
|
|
region_name=src_dict.get('OS_REGION_NAME', None),
|
|
endpoint_type=src_dict.get('OS_ENDPOINT_TYPE', None)
|
|
)
|
|
|
|
|
|
def create_dir_tree(dir):
|
|
try:
|
|
os.makedirs(dir)
|
|
except OSError as exc:
|
|
if exc.errno == errno.EEXIST and os.path.isdir(dir):
|
|
pass
|
|
else:
|
|
raise exc
|
|
|
|
|
|
def create_dir(directory, do_log=True):
|
|
"""
|
|
Creates a directory if it doesn't exists and write the execution
|
|
in the logs
|
|
"""
|
|
expanded_dir_name = os.path.expanduser(directory)
|
|
try:
|
|
if not os.path.isdir(expanded_dir_name):
|
|
if do_log:
|
|
logging.warning('[*] Directory {0} does not exists,\
|
|
creating...'.format(expanded_dir_name))
|
|
os.makedirs(expanded_dir_name)
|
|
else:
|
|
if do_log:
|
|
logging.warning('[*] Directory {0} found!'.format(
|
|
expanded_dir_name))
|
|
except Exception as error:
|
|
err = '[*] Error while creating directory {0}: {1}\
|
|
'.format(expanded_dir_name, error)
|
|
raise Exception(err)
|
|
|
|
|
|
def save_config_to_file(config, f, section='freezer_default'):
|
|
parser = configparser.ConfigParser()
|
|
parser.add_section(section)
|
|
for option, option_value in config.items():
|
|
parser.set(section, option, option_value)
|
|
parser.write(f)
|
|
f.close()
|
|
|
|
|
|
class DateTime(object):
|
|
def __init__(self, value):
|
|
if isinstance(value, int):
|
|
self.date_time = datetime.datetime.fromtimestamp(value)
|
|
elif isinstance(value, datetime.datetime):
|
|
self.date_time = value
|
|
else:
|
|
fmt = '%Y-%m-%dT%H:%M:%S'
|
|
try:
|
|
self.date_time = datetime.datetime.strptime(value, fmt)
|
|
except Exception:
|
|
raise Exception('bad datetime format: "{0}'.format(value))
|
|
|
|
@property
|
|
def timestamp(self):
|
|
return int(time.mktime(self.date_time.timetuple()))
|
|
|
|
def __repr__(self):
|
|
return self.date_time.strftime('%Y-%m-%d %H:%M:%S')
|
|
|
|
def __sub__(self, other):
|
|
assert isinstance(other, DateTime)
|
|
return self.date_time - other.date_time # return timedelta
|
|
|
|
@staticmethod
|
|
def now():
|
|
return DateTime(datetime.datetime.now())
|
|
|
|
|
|
def get_vol_fs_type(vol_name):
|
|
"""
|
|
The argument need to be a full path lvm name i.e. /dev/vg0/var
|
|
or a disk partition like /dev/sda1. The returnet value is the
|
|
file system type
|
|
"""
|
|
if os.path.exists(vol_name) is False:
|
|
err = '[*] Provided volume name not found: {0} '.format(vol_name)
|
|
logging.exception(err)
|
|
raise Exception(err)
|
|
|
|
file_cmd = '{0} -0 -bLs --no-pad --no-buffer --preserve-date \
|
|
{1}'.format(find_executable("file"), vol_name)
|
|
file_process = subprocess.Popen(
|
|
file_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, shell=True,
|
|
executable=find_executable("bash"))
|
|
(file_out, file_err) = file_process.communicate()
|
|
file_match = re.search(r'(\S+?) filesystem data', file_out, re.I)
|
|
if file_match is None:
|
|
err = '[*] File system type not guessable: {0}'.format(file_err)
|
|
logging.exception(err)
|
|
raise Exception(err)
|
|
else:
|
|
filesys_type = file_match.group(1)
|
|
logging.info('[*] File system {0} found for volume {1}'.format(
|
|
filesys_type, vol_name))
|
|
return filesys_type.lower().strip()
|
|
|
|
|
|
def path_join(*args):
|
|
"""Should work for windows and linux"""
|
|
return "/".join([str(x) for x in args])
|
|
|
|
|
|
def get_mount_from_path(path):
|
|
"""
|
|
Take a file system path as argument and return the mount point
|
|
for that file system path.
|
|
|
|
:param path: file system path
|
|
:returns: mount point of path, rest of the path
|
|
"""
|
|
|
|
if not os.path.exists(path):
|
|
logging.critical('[*] Error: provided path does not exist: {0}'
|
|
.format(path))
|
|
raise IOError
|
|
|
|
mount_point_path = os.path.abspath(path)
|
|
|
|
while not os.path.ismount(mount_point_path):
|
|
mount_point_path = os.path.dirname(mount_point_path)
|
|
return mount_point_path, os.path.relpath(path, mount_point_path)
|
|
|
|
|
|
# see: http://goo.gl/kTQMs
|
|
HUMAN_2_SYMBOLS = {
|
|
'customary': ('B', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'),
|
|
'customary_ext': ('byte', 'kilo', 'mega', 'giga', 'tera', 'peta', 'exa',
|
|
'zetta', 'iotta'),
|
|
'iec': ('Bi', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi'),
|
|
'iec_ext': ('byte', 'kibi', 'mebi', 'gibi', 'tebi', 'pebi', 'exbi',
|
|
'zebi', 'yobi'),
|
|
}
|
|
|
|
|
|
def human2bytes(s):
|
|
"""
|
|
Attempts to guess the string format based on default symbols
|
|
set and return the corresponding bytes as an integer.
|
|
When unable to recognize the format ValueError is raised.
|
|
"""
|
|
if s.isdigit():
|
|
return int(s)
|
|
|
|
if s in (False, None, '-1'):
|
|
return -1
|
|
|
|
init = s
|
|
num = ""
|
|
while s and s[0:1].isdigit() or s[0:1] == '.':
|
|
num += s[0]
|
|
s = s[1:]
|
|
num = float(num)
|
|
letter = s.strip()
|
|
for name, sset in HUMAN_2_SYMBOLS.items():
|
|
if letter in sset:
|
|
break
|
|
else:
|
|
if letter == 'k':
|
|
# treat 'k' as an alias for 'K' as per: http://goo.gl/kTQMs
|
|
sset = HUMAN_2_SYMBOLS['customary']
|
|
letter = letter.upper()
|
|
else:
|
|
raise ValueError("can't interpret %r" % init)
|
|
prefix = {sset[0]: 1}
|
|
for i, s in enumerate(sset[1:]):
|
|
prefix[s] = 1 << (i + 1) * 10
|
|
return int(num * prefix[letter])
|
|
|
|
|
|
def create_subprocess(cmd):
|
|
"""
|
|
Create a new subprocess in the OS
|
|
:param cmd: command to execute in the subprocess
|
|
:return: the output and errors of the subprocess
|
|
"""
|
|
process = subprocess.Popen(cmd,
|
|
stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE)
|
|
return process.communicate()
|
|
|
|
|
|
def date_to_timestamp(date):
|
|
fmt = '%Y-%m-%dT%H:%M:%S'
|
|
opt_backup_date = datetime.datetime.strptime(date, fmt)
|
|
return int(time.mktime(opt_backup_date.timetuple()))
|
|
|
|
|
|
class Bunch:
|
|
def __init__(self, **kwds):
|
|
self.__dict__.update(kwds)
|
|
|
|
def __getattr__(self, item):
|
|
return self.__dict__.get(item)
|
|
|
|
|
|
class ReSizeStream:
|
|
"""
|
|
Iterator/File-like object for changing size of chunk in stream
|
|
"""
|
|
def __init__(self, stream, length, chunk_size):
|
|
self.stream = stream
|
|
self.length = length
|
|
self.chunk_size = chunk_size
|
|
self.reminder = ""
|
|
self.transmitted = 0
|
|
|
|
def __len__(self):
|
|
return self.length
|
|
|
|
def __iter__(self):
|
|
return self
|
|
|
|
def next(self):
|
|
logging.info("Transmitted {0} of {1}".format(self.transmitted,
|
|
self.length))
|
|
chunk_size = self.chunk_size
|
|
if len(self.reminder) > chunk_size:
|
|
result = self.reminder[:chunk_size]
|
|
self.reminder = self.reminder[chunk_size:]
|
|
self.transmitted += len(result)
|
|
return result
|
|
else:
|
|
stop = False
|
|
while not stop and len(self.reminder) < chunk_size:
|
|
try:
|
|
self.reminder += next(self.stream)
|
|
except StopIteration:
|
|
stop = True
|
|
if stop:
|
|
result = self.reminder
|
|
if len(self.reminder) == 0:
|
|
raise StopIteration()
|
|
self.reminder = []
|
|
self.transmitted += len(result)
|
|
return result
|
|
else:
|
|
result = self.reminder[:chunk_size]
|
|
self.reminder = self.reminder[chunk_size:]
|
|
self.transmitted += len(result)
|
|
return result
|
|
|
|
def read(self, chunk_size):
|
|
self.chunk_size = chunk_size
|
|
return self.next()
|
|
|
|
|
|
def dequote(s):
|
|
"""
|
|
If a string has single or double quotes around it, remove them.
|
|
Make sure the pair of quotes match.
|
|
If a matching pair of quotes is not found, return the string unchanged.
|
|
"""
|
|
s = s.rstrip('\n')
|
|
if (s[0] == s[-1]) and s.startswith(("'", '"')):
|
|
return s[1:-1]
|
|
return s
|
|
|
|
|
|
def find_executable(name):
|
|
return distspawn.find_executable(name)
|
|
|
|
|
|
def openssl_path():
|
|
from freezer import winutils
|
|
if winutils.is_windows():
|
|
return 'openssl'
|
|
else:
|
|
return find_executable('openssl')
|
|
|
|
|
|
def tar_path():
|
|
"""This function returns tar binary path"""
|
|
from freezer import winutils
|
|
if winutils.is_windows():
|
|
path_to_binaries = os.path.dirname(os.path.abspath(__file__))
|
|
return '{0}\\bin\\tar.exe'.format(path_to_binaries)
|
|
|
|
tar = (get_executable_path('gnutar') or get_executable_path('gtar')
|
|
or get_executable_path('tar'))
|
|
if not tar:
|
|
raise Exception('Please install gnu tar (gtar) as it is a '
|
|
'mandatory requirement to use freezer.')
|
|
return tar
|
|
|
|
|
|
def get_executable_path(binary):
|
|
"""
|
|
This function returns the executable path of a given binary
|
|
if it is found in the system.
|
|
:param binary:
|
|
:type binary: str
|
|
:rtype: str
|
|
:return: Absoulte Path to the executable file
|
|
"""
|
|
from freezer import winutils
|
|
if winutils.is_windows():
|
|
path_to_binaries = os.path.dirname(os.path.abspath(__file__))
|
|
return '{0}\\bin\\{1}.exe'.format(path_to_binaries, binary)
|
|
|
|
elif is_bsd():
|
|
return (distspawn.find_executable(binary) or
|
|
distspawn.find_executable(binary, path=':'.join(sys.path)))
|
|
else:
|
|
return distspawn.find_executable(binary)
|
|
|
|
|
|
def alter_proxy(proxy):
|
|
"""
|
|
Read proxy option from dictionary and alter the HTTP_PROXY and/or
|
|
HTTPS_PROXY system variables
|
|
"""
|
|
# Default case where 'proxy' key is not set -- do nothing
|
|
proxy_value = proxy.lower()
|
|
# python-swift client takes into account both
|
|
# upper and lower case proxies so clear them all
|
|
os.environ.pop("http_proxy", None)
|
|
os.environ.pop("https_proxy", None)
|
|
os.environ.pop("HTTP_PROXY", None)
|
|
os.environ.pop("HTTPS_PROXY", None)
|
|
if proxy_value.startswith('http://') or \
|
|
proxy_value.startswith('https://'):
|
|
logging.info('[*] Using proxy {0}'.format(proxy_value))
|
|
os.environ['HTTP_PROXY'] = str(proxy_value)
|
|
os.environ['HTTPS_PROXY'] = str(proxy_value)
|
|
else:
|
|
raise Exception('Proxy has unknown scheme')
|
|
|
|
def is_bsd():
|
|
return 'darwin' in sys.platform or 'bsd' in sys.platform
|
|
|
|
|
|
def shield(func):
|
|
"""Remove try except boilerplate code from functions"""
|
|
@wraps(func)
|
|
def wrapper(*args, **kwargs):
|
|
try:
|
|
return func(*args, **kwargs)
|
|
except Exception as error:
|
|
logging.error(error)
|
|
return wrapper
|
|
|
|
|
|
def delete_file(path_to_file):
|
|
"""Delete a file from the file system
|
|
"""
|
|
try:
|
|
os.remove(path_to_file)
|
|
except Exception:
|
|
logging.warning("Error deleting file {0}".format(path_to_file))
|
|
|
|
|
|
class Namespace(dict):
|
|
"""A dict subclass that exposes its items as attributes.
|
|
|
|
Warning: Namespace instances do not have direct access to the
|
|
dict methods.
|
|
|
|
"""
|
|
|
|
def __init__(self, obj={}):
|
|
super(Namespace, self).__init__(obj)
|
|
|
|
def __dir__(self):
|
|
return tuple(self)
|
|
|
|
def __repr__(self):
|
|
return "%s(%s)" % (type(self).__name__,
|
|
super(Namespace, self).__repr__())
|
|
|
|
def __getattribute__(self, name):
|
|
try:
|
|
return self[name]
|
|
except KeyError:
|
|
# Return None in case the value doesn't exists
|
|
# this is not an issue for the apiclient because it skips
|
|
# None values
|
|
return None
|
|
|
|
def __setattr__(self, name, value):
|
|
self[name] = value
|
|
|
|
def __delattr__(self, name):
|
|
del self[name]
|
|
|
|
@classmethod
|
|
def from_object(cls, obj, names=None):
|
|
if names is None:
|
|
names = dir(obj)
|
|
ns = {name:getattr(obj, name) for name in names}
|
|
return cls(ns)
|
|
|
|
@classmethod
|
|
def from_mapping(cls, ns, names=None):
|
|
if names:
|
|
ns = {name: ns[name] for name in names}
|
|
return cls(ns)
|
|
|
|
@classmethod
|
|
def from_sequence(cls, seq, names=None):
|
|
if names:
|
|
seq = {name: val for name, val in seq if name in names}
|
|
return cls(seq)
|
|
|
|
@staticmethod
|
|
def hasattr(ns, name):
|
|
try:
|
|
object.__getattribute__(ns, name)
|
|
except AttributeError:
|
|
return False
|
|
return True
|
|
|
|
@staticmethod
|
|
def getattr(ns, name):
|
|
return object.__getattribute__(ns, name)
|
|
|
|
@staticmethod
|
|
def setattr(ns, name, value):
|
|
return object.__setattr__(ns, name, value)
|
|
|
|
@staticmethod
|
|
def delattr(ns, name):
|
|
return object.__delattr__(ns, name)
|