8f4b53da68
And pep8 fixing (suddenly this pull request was rejected by jenkins and pep8. so this commit also contains some small style changes for pep8.) Currently the default value for logging is INFO and it is hardcoded. Additional command will be helpful for debug/development process. Implements blueprint: logging-level-cli Change-Id: I1105e3da277e2a948f3ff50b04751169eddf26ed
518 lines
16 KiB
Python
518 lines
16 KiB
Python
"""
|
|
(c) Copyright 2014,2015 Hewlett-Packard Development Company, L.P.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
|
|
Freezer general utils functions
|
|
"""
|
|
import datetime
|
|
import errno
|
|
import logging
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
from ConfigParser import ConfigParser
|
|
from distutils import spawn as distspawn
|
|
from functools import wraps
|
|
|
|
|
|
class OpenstackOptions:
|
|
"""
|
|
Stores credentials for OpenStack API.
|
|
Can be created using
|
|
>> create_from_env()
|
|
or
|
|
>> create_from_dict(dict)
|
|
"""
|
|
def __init__(self, user_name, tenant_name, project_name, auth_url,
|
|
password, identity_api_version, tenant_id=None,
|
|
region_name=None, endpoint_type=None):
|
|
self.user_name = user_name
|
|
self.tenant_name = tenant_name
|
|
self.auth_url = auth_url
|
|
self.password = password
|
|
self.tenant_id = tenant_id
|
|
self.project_name = project_name
|
|
self.identity_api_version = identity_api_version
|
|
self.region_name = region_name
|
|
self.endpoint_type = endpoint_type
|
|
if not (self.password and self.user_name and self.auth_url and
|
|
(self.tenant_name or self.project_name)):
|
|
raise Exception("Please set up in your env:"
|
|
"OS_USERNAME, OS_TENANT_NAME/OS_PROJECT_NAME,"
|
|
" OS_AUTH_URL, OS_PASSWORD")
|
|
|
|
@property
|
|
def os_options(self):
|
|
"""
|
|
:return: The OpenStack options which can have tenant_id,
|
|
auth_token, service_type, endpoint_type, tenant_name,
|
|
object_storage_url, region_name
|
|
"""
|
|
return {'tenant_id': self.tenant_id,
|
|
'tenant_name': self.tenant_name,
|
|
'project_name': self.project_name,
|
|
'identity_api_version': self.identity_api_version,
|
|
'region_name': self.region_name,
|
|
'endpoint_type': self.endpoint_type}
|
|
|
|
@staticmethod
|
|
def create_from_env():
|
|
return OpenstackOptions.create_from_dict(os.environ)
|
|
|
|
@staticmethod
|
|
def create_from_dict(src_dict):
|
|
return OpenstackOptions(
|
|
user_name=src_dict.get('OS_USERNAME', None),
|
|
tenant_name=src_dict.get('OS_TENANT_NAME', None),
|
|
project_name=src_dict.get('OS_PROJECT_NAME', None),
|
|
auth_url=src_dict.get('OS_AUTH_URL', None),
|
|
identity_api_version=src_dict.get('OS_IDENTITY_API_VERSION',
|
|
'2.0'),
|
|
password=src_dict.get('OS_PASSWORD', None),
|
|
tenant_id=src_dict.get('OS_TENANT_ID', None),
|
|
region_name=src_dict.get('OS_REGION_NAME', None),
|
|
endpoint_type=src_dict.get('OS_ENDPOINT_TYPE', None)
|
|
)
|
|
|
|
|
|
def create_dir_tree(dir):
|
|
try:
|
|
os.makedirs(dir)
|
|
except OSError as exc:
|
|
if exc.errno == errno.EEXIST and os.path.isdir(dir):
|
|
pass
|
|
else:
|
|
raise exc
|
|
|
|
|
|
def create_dir(directory, do_log=True):
|
|
"""
|
|
Creates a directory if it doesn't exists and write the execution
|
|
in the logs
|
|
"""
|
|
expanded_dir_name = os.path.expanduser(directory)
|
|
try:
|
|
if not os.path.isdir(expanded_dir_name):
|
|
if do_log:
|
|
logging.warning('[*] Directory {0} does not exists,\
|
|
creating...'.format(expanded_dir_name))
|
|
os.makedirs(expanded_dir_name)
|
|
else:
|
|
if do_log:
|
|
logging.warning('[*] Directory {0} found!'.format(
|
|
expanded_dir_name))
|
|
except Exception as error:
|
|
err = '[*] Error while creating directory {0}: {1}\
|
|
'.format(expanded_dir_name, error)
|
|
raise Exception(err)
|
|
|
|
|
|
def save_config_to_file(config, f, section='freezer_default'):
|
|
parser = ConfigParser()
|
|
parser.add_section(section)
|
|
for option, option_value in config.items():
|
|
parser.set(section, option, option_value)
|
|
parser.write(f)
|
|
f.close()
|
|
|
|
|
|
class DateTime(object):
|
|
def __init__(self, value):
|
|
if isinstance(value, int):
|
|
self.date_time = datetime.datetime.fromtimestamp(value)
|
|
elif isinstance(value, datetime.datetime):
|
|
self.date_time = value
|
|
else:
|
|
fmt = '%Y-%m-%dT%H:%M:%S'
|
|
try:
|
|
self.date_time = datetime.datetime.strptime(value, fmt)
|
|
except Exception:
|
|
raise Exception('bad datetime format: "{0}'.format(value))
|
|
|
|
@property
|
|
def timestamp(self):
|
|
return int(time.mktime(self.date_time.timetuple()))
|
|
|
|
def __repr__(self):
|
|
return self.date_time.strftime('%Y-%m-%d %H:%M:%S')
|
|
|
|
def __sub__(self, other):
|
|
assert isinstance(other, DateTime)
|
|
return self.date_time - other.date_time # return timedelta
|
|
|
|
@staticmethod
|
|
def now():
|
|
return DateTime(datetime.datetime.now())
|
|
|
|
|
|
def get_vol_fs_type(vol_name):
|
|
"""
|
|
The argument need to be a full path lvm name i.e. /dev/vg0/var
|
|
or a disk partition like /dev/sda1. The returnet value is the
|
|
file system type
|
|
"""
|
|
if os.path.exists(vol_name) is False:
|
|
err = '[*] Provided volume name not found: {0} '.format(vol_name)
|
|
logging.exception(err)
|
|
raise Exception(err)
|
|
|
|
file_cmd = '{0} -0 -bLs --no-pad --no-buffer --preserve-date \
|
|
{1}'.format(find_executable("file"), vol_name)
|
|
file_process = subprocess.Popen(
|
|
file_cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE, shell=True,
|
|
executable=find_executable("bash"))
|
|
(file_out, file_err) = file_process.communicate()
|
|
file_match = re.search(r'(\S+?) filesystem data', file_out, re.I)
|
|
if file_match is None:
|
|
err = '[*] File system type not guessable: {0}'.format(file_err)
|
|
logging.exception(err)
|
|
raise Exception(err)
|
|
else:
|
|
filesys_type = file_match.group(1)
|
|
logging.info('[*] File system {0} found for volume {1}'.format(
|
|
filesys_type, vol_name))
|
|
return filesys_type.lower().strip()
|
|
|
|
|
|
def path_join(*args):
|
|
"""Should work for windows and linux"""
|
|
return "/".join([str(x) for x in args])
|
|
|
|
|
|
def get_mount_from_path(path):
|
|
"""
|
|
Take a file system path as argument and return the mount point
|
|
for that file system path.
|
|
|
|
:param path: file system path
|
|
:returns: mount point of path, rest of the path
|
|
"""
|
|
|
|
if not os.path.exists(path):
|
|
logging.critical('[*] Error: provided path does not exist: {0}'
|
|
.format(path))
|
|
raise IOError
|
|
|
|
mount_point_path = os.path.abspath(path)
|
|
|
|
while not os.path.ismount(mount_point_path):
|
|
mount_point_path = os.path.dirname(mount_point_path)
|
|
return mount_point_path, os.path.relpath(path, mount_point_path)
|
|
|
|
|
|
# see: http://goo.gl/kTQMs
|
|
HUMAN_2_SYMBOLS = {
|
|
'customary': ('B', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'),
|
|
'customary_ext': ('byte', 'kilo', 'mega', 'giga', 'tera', 'peta', 'exa',
|
|
'zetta', 'iotta'),
|
|
'iec': ('Bi', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi'),
|
|
'iec_ext': ('byte', 'kibi', 'mebi', 'gibi', 'tebi', 'pebi', 'exbi',
|
|
'zebi', 'yobi'),
|
|
}
|
|
|
|
|
|
def human2bytes(s):
|
|
"""
|
|
Attempts to guess the string format based on default symbols
|
|
set and return the corresponding bytes as an integer.
|
|
When unable to recognize the format ValueError is raised.
|
|
"""
|
|
if s.isdigit():
|
|
return long(s)
|
|
|
|
if s in (False, None, '-1'):
|
|
return -1
|
|
|
|
init = s
|
|
num = ""
|
|
while s and s[0:1].isdigit() or s[0:1] == '.':
|
|
num += s[0]
|
|
s = s[1:]
|
|
num = float(num)
|
|
letter = s.strip()
|
|
for name, sset in HUMAN_2_SYMBOLS.items():
|
|
if letter in sset:
|
|
break
|
|
else:
|
|
if letter == 'k':
|
|
# treat 'k' as an alias for 'K' as per: http://goo.gl/kTQMs
|
|
sset = HUMAN_2_SYMBOLS['customary']
|
|
letter = letter.upper()
|
|
else:
|
|
raise ValueError("can't interpret %r" % init)
|
|
prefix = {sset[0]: 1}
|
|
for i, s in enumerate(sset[1:]):
|
|
prefix[s] = 1 << (i + 1) * 10
|
|
return int(num * prefix[letter])
|
|
|
|
|
|
def create_subprocess(cmd):
|
|
"""
|
|
Create a new subprocess in the OS
|
|
:param cmd: command to execute in the subprocess
|
|
:return: the output and errors of the subprocess
|
|
"""
|
|
process = subprocess.Popen(cmd,
|
|
stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE)
|
|
return process.communicate()
|
|
|
|
|
|
def date_to_timestamp(date):
|
|
fmt = '%Y-%m-%dT%H:%M:%S'
|
|
opt_backup_date = datetime.datetime.strptime(date, fmt)
|
|
return int(time.mktime(opt_backup_date.timetuple()))
|
|
|
|
|
|
class Bunch:
|
|
def __init__(self, **kwds):
|
|
self.__dict__.update(kwds)
|
|
|
|
def __getattr__(self, item):
|
|
return self.__dict__.get(item)
|
|
|
|
|
|
class ReSizeStream:
|
|
"""
|
|
Iterator/File-like object for changing size of chunk in stream
|
|
"""
|
|
def __init__(self, stream, length, chunk_size):
|
|
self.stream = stream
|
|
self.length = length
|
|
self.chunk_size = chunk_size
|
|
self.reminder = ""
|
|
self.transmitted = 0
|
|
|
|
def __len__(self):
|
|
return self.length
|
|
|
|
def __iter__(self):
|
|
return self
|
|
|
|
def next(self):
|
|
logging.info("Transmitted {0} of {1}".format(self.transmitted,
|
|
self.length))
|
|
chunk_size = self.chunk_size
|
|
if len(self.reminder) > chunk_size:
|
|
result = self.reminder[:chunk_size]
|
|
self.reminder = self.reminder[chunk_size:]
|
|
self.transmitted += len(result)
|
|
return result
|
|
else:
|
|
stop = False
|
|
while not stop and len(self.reminder) < chunk_size:
|
|
try:
|
|
self.reminder += next(self.stream)
|
|
except StopIteration:
|
|
stop = True
|
|
if stop:
|
|
result = self.reminder
|
|
if len(self.reminder) == 0:
|
|
raise StopIteration()
|
|
self.reminder = []
|
|
self.transmitted += len(result)
|
|
return result
|
|
else:
|
|
result = self.reminder[:chunk_size]
|
|
self.reminder = self.reminder[chunk_size:]
|
|
self.transmitted += len(result)
|
|
return result
|
|
|
|
def read(self, chunk_size):
|
|
self.chunk_size = chunk_size
|
|
return self.next()
|
|
|
|
|
|
def dequote(s):
|
|
"""
|
|
If a string has single or double quotes around it, remove them.
|
|
Make sure the pair of quotes match.
|
|
If a matching pair of quotes is not found, return the string unchanged.
|
|
"""
|
|
s = s.rstrip('\n')
|
|
if (s[0] == s[-1]) and s.startswith(("'", '"')):
|
|
return s[1:-1]
|
|
return s
|
|
|
|
|
|
def find_executable(name):
|
|
return distspawn.find_executable(name)
|
|
|
|
|
|
def openssl_path():
|
|
import winutils
|
|
if winutils.is_windows():
|
|
return 'openssl'
|
|
else:
|
|
return find_executable('openssl')
|
|
|
|
|
|
def tar_path():
|
|
"""This function returns tar binary path"""
|
|
from winutils import is_windows
|
|
if is_windows():
|
|
path_to_binaries = os.path.dirname(os.path.abspath(__file__))
|
|
return '{0}\\bin\\tar.exe'.format(path_to_binaries)
|
|
|
|
tar = (get_executable_path('gnutar') or get_executable_path('gtar')
|
|
or get_executable_path('tar'))
|
|
if not tar:
|
|
raise Exception('Please install gnu tar (gtar) as it is a '
|
|
'mandatory requirement to use freezer.')
|
|
return tar
|
|
|
|
|
|
def get_executable_path(binary):
|
|
"""
|
|
This function returns the executable path of a given binary
|
|
if it is found in the system.
|
|
:param binary:
|
|
:type binary: str
|
|
:rtype: str
|
|
:return: Absoulte Path to the executable file
|
|
"""
|
|
from winutils import is_windows
|
|
if is_windows():
|
|
path_to_binaries = os.path.dirname(os.path.abspath(__file__))
|
|
return '{0}\\bin\\{1}.exe'.format(path_to_binaries, binary)
|
|
|
|
elif is_bsd():
|
|
return (distspawn.find_executable(binary) or
|
|
distspawn.find_executable(binary, path=':'.join(sys.path)))
|
|
else:
|
|
return distspawn.find_executable(binary)
|
|
|
|
|
|
def alter_proxy(proxy):
|
|
"""
|
|
Read proxy option from dictionary and alter the HTTP_PROXY and/or
|
|
HTTPS_PROXY system variables
|
|
"""
|
|
# Default case where 'proxy' key is not set -- do nothing
|
|
proxy_value = proxy.lower()
|
|
# python-swift client takes into account both
|
|
# upper and lower case proxies so clear them all
|
|
os.environ.pop("http_proxy", None)
|
|
os.environ.pop("https_proxy", None)
|
|
os.environ.pop("HTTP_PROXY", None)
|
|
os.environ.pop("HTTPS_PROXY", None)
|
|
if proxy_value.startswith('http://') or \
|
|
proxy_value.startswith('https://'):
|
|
logging.info('[*] Using proxy {0}'.format(proxy_value))
|
|
os.environ['HTTP_PROXY'] = str(proxy_value)
|
|
os.environ['HTTPS_PROXY'] = str(proxy_value)
|
|
else:
|
|
raise Exception('Proxy has unknown scheme')
|
|
|
|
def is_bsd():
|
|
return 'darwin' in sys.platform or 'bsd' in sys.platform
|
|
|
|
|
|
def shield(func):
|
|
"""Remove try except boilerplate code from functions"""
|
|
@wraps(func)
|
|
def wrapper(*args, **kwargs):
|
|
try:
|
|
return func(*args, **kwargs)
|
|
except Exception as error:
|
|
logging.error(error)
|
|
return wrapper
|
|
|
|
|
|
def delete_file(path_to_file):
|
|
"""Delete a file from the file system
|
|
"""
|
|
try:
|
|
os.remove(path_to_file)
|
|
except Exception:
|
|
logging.warning("Error deleting file {0}".format(path_to_file))
|
|
|
|
|
|
class Namespace(dict):
|
|
"""A dict subclass that exposes its items as attributes.
|
|
|
|
Warning: Namespace instances do not have direct access to the
|
|
dict methods.
|
|
|
|
"""
|
|
|
|
def __init__(self, obj={}):
|
|
super(Namespace, self).__init__(obj)
|
|
|
|
def __dir__(self):
|
|
return tuple(self)
|
|
|
|
def __repr__(self):
|
|
return "%s(%s)" % (type(self).__name__,
|
|
super(Namespace, self).__repr__())
|
|
|
|
def __getattribute__(self, name):
|
|
try:
|
|
return self[name]
|
|
except KeyError:
|
|
# Return None in case the value doesn't exists
|
|
# this is not an issue for the apiclient because it skips
|
|
# None values
|
|
return None
|
|
|
|
def __setattr__(self, name, value):
|
|
self[name] = value
|
|
|
|
def __delattr__(self, name):
|
|
del self[name]
|
|
|
|
@classmethod
|
|
def from_object(cls, obj, names=None):
|
|
if names is None:
|
|
names = dir(obj)
|
|
ns = {name:getattr(obj, name) for name in names}
|
|
return cls(ns)
|
|
|
|
@classmethod
|
|
def from_mapping(cls, ns, names=None):
|
|
if names:
|
|
ns = {name: ns[name] for name in names}
|
|
return cls(ns)
|
|
|
|
@classmethod
|
|
def from_sequence(cls, seq, names=None):
|
|
if names:
|
|
seq = {name: val for name, val in seq if name in names}
|
|
return cls(seq)
|
|
|
|
@staticmethod
|
|
def hasattr(ns, name):
|
|
try:
|
|
object.__getattribute__(ns, name)
|
|
except AttributeError:
|
|
return False
|
|
return True
|
|
|
|
@staticmethod
|
|
def getattr(ns, name):
|
|
return object.__getattribute__(ns, name)
|
|
|
|
@staticmethod
|
|
def setattr(ns, name, value):
|
|
return object.__setattr__(ns, name, value)
|
|
|
|
@staticmethod
|
|
def delattr(ns, name):
|
|
return object.__delattr__(ns, name)
|