442 lines
14 KiB
Python
442 lines
14 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
|
|
|
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
|
|
# Copyright (C) 2013 Rackspace Hosting All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import collections
|
|
import copy
|
|
import errno
|
|
import functools
|
|
import logging
|
|
import os
|
|
import six
|
|
import sys
|
|
import traceback
|
|
|
|
from taskflow import exceptions
|
|
from taskflow.utils import reflection
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
NUMERIC_TYPES = tuple(list(six.integer_types) + [float])
|
|
|
|
|
|
def wraps(fn):
|
|
"""This will not be needed in python 3.2 or greater which already has this
|
|
built-in to its functools.wraps method.
|
|
"""
|
|
|
|
def wrapper(f):
|
|
f = functools.wraps(fn)(f)
|
|
f.__wrapped__ = getattr(fn, '__wrapped__', fn)
|
|
return f
|
|
|
|
return wrapper
|
|
|
|
|
|
def get_version_string(obj):
|
|
"""Gets a object's version as a string.
|
|
|
|
Returns string representation of object's version taken from
|
|
its 'version' attribute, or None if object does not have such
|
|
attribute or its version is None.
|
|
"""
|
|
obj_version = getattr(obj, 'version', None)
|
|
if isinstance(obj_version, (list, tuple)):
|
|
obj_version = '.'.join(str(item) for item in obj_version)
|
|
if obj_version is not None and not isinstance(obj_version,
|
|
six.string_types):
|
|
obj_version = str(obj_version)
|
|
return obj_version
|
|
|
|
|
|
def get_duplicate_keys(iterable, key=None):
|
|
if key is not None:
|
|
iterable = six.moves.map(key, iterable)
|
|
keys = set()
|
|
duplicates = set()
|
|
for item in iterable:
|
|
if item in keys:
|
|
duplicates.add(item)
|
|
keys.add(item)
|
|
return duplicates
|
|
|
|
|
|
def is_valid_attribute_name(name, allow_self=False, allow_hidden=False):
|
|
"""Validates that a string name is a valid/invalid python attribute name"""
|
|
if not isinstance(name, six.string_types) or len(name) == 0:
|
|
return False
|
|
# Make the name just be a simple string in latin-1 encoding in python3
|
|
name = six.b(name)
|
|
if not allow_self and name.lower().startswith(six.b('self')):
|
|
return False
|
|
if not allow_hidden and name.startswith(six.b("_")):
|
|
return False
|
|
# See: http://docs.python.org/release/2.5.2/ref/grammar.txt (or newer)
|
|
#
|
|
# Python identifiers should start with a letter.
|
|
if isinstance(name[0], six.integer_types):
|
|
if not chr(name[0]).isalpha():
|
|
return False
|
|
else:
|
|
if not name[0].isalpha():
|
|
return False
|
|
for i in range(1, len(name)):
|
|
symbol = name[i]
|
|
# The rest of an attribute name follows: (letter | digit | "_")*
|
|
if isinstance(symbol, six.integer_types):
|
|
symbol = chr(symbol)
|
|
if not (symbol.isalpha() or symbol.isdigit() or symbol == "_"):
|
|
return False
|
|
else:
|
|
if not (symbol.isalpha() or symbol.isdigit() or symbol == "_"):
|
|
return False
|
|
return True
|
|
|
|
|
|
class AttrDict(dict):
|
|
"""Helper utility dict sub-class to create a class that can be accessed by
|
|
attribute name from a dictionary that contains a set of keys and values.
|
|
"""
|
|
NO_ATTRS = tuple(reflection.get_member_names(dict))
|
|
|
|
@classmethod
|
|
def _is_valid_attribute_name(cls, name):
|
|
if not is_valid_attribute_name(name):
|
|
return False
|
|
# Make the name just be a simple string in latin-1 encoding in python3
|
|
if name in cls.NO_ATTRS:
|
|
return False
|
|
return True
|
|
|
|
def __init__(self, **kwargs):
|
|
for (k, v) in kwargs.items():
|
|
if not self._is_valid_attribute_name(k):
|
|
raise AttributeError("Invalid attribute name: '%s'" % (k))
|
|
self[k] = v
|
|
|
|
def __getattr__(self, name):
|
|
if not self._is_valid_attribute_name(name):
|
|
raise AttributeError("Invalid attribute name: '%s'" % (name))
|
|
try:
|
|
return self[name]
|
|
except KeyError:
|
|
raise AttributeError("No attributed named: '%s'" % (name))
|
|
|
|
def __setattr__(self, name, value):
|
|
if not self._is_valid_attribute_name(name):
|
|
raise AttributeError("Invalid attribute name: '%s'" % (name))
|
|
self[name] = value
|
|
|
|
|
|
class ExponentialBackoff(object):
|
|
"""An iterable object that will yield back an exponential delay sequence
|
|
provided an exponent and a number of items to yield. This object may be
|
|
iterated over multiple times (yielding the same sequence each time).
|
|
"""
|
|
def __init__(self, attempts, exponent=2):
|
|
self.attempts = int(attempts)
|
|
self.exponent = exponent
|
|
|
|
def __iter__(self):
|
|
if self.attempts <= 0:
|
|
raise StopIteration()
|
|
for i in xrange(0, self.attempts):
|
|
yield self.exponent ** i
|
|
|
|
def __str__(self):
|
|
return "ExponentialBackoff: %s" % ([str(v) for v in self])
|
|
|
|
|
|
def as_bool(val):
|
|
"""Converts an arbitary value into a boolean."""
|
|
if isinstance(val, bool):
|
|
return val
|
|
if isinstance(val, six.string_types):
|
|
if val.lower() in ('f', 'false', '0', 'n', 'no'):
|
|
return False
|
|
if val.lower() in ('t', 'true', '1', 'y', 'yes'):
|
|
return True
|
|
return bool(val)
|
|
|
|
|
|
def as_int(obj, quiet=False):
|
|
"""Converts an arbitary value into a integer."""
|
|
# Try "2" -> 2
|
|
try:
|
|
return int(obj)
|
|
except (ValueError, TypeError):
|
|
pass
|
|
# Try "2.5" -> 2
|
|
try:
|
|
return int(float(obj))
|
|
except (ValueError, TypeError):
|
|
pass
|
|
# Eck, not sure what this is then.
|
|
if not quiet:
|
|
raise TypeError("Can not translate %s to an integer." % (obj))
|
|
return obj
|
|
|
|
|
|
# Taken from oslo-incubator file-utils but since that module pulls in a large
|
|
# amount of other files it does not seem so useful to include that full
|
|
# module just for this function.
|
|
def ensure_tree(path):
|
|
"""Create a directory (and any ancestor directories required)
|
|
|
|
:param path: Directory to create
|
|
"""
|
|
try:
|
|
os.makedirs(path)
|
|
except OSError as exc:
|
|
if exc.errno == errno.EEXIST:
|
|
if not os.path.isdir(path):
|
|
raise
|
|
else:
|
|
raise
|
|
|
|
|
|
class TransitionNotifier(object):
|
|
"""A utility helper class that can be used to subscribe to
|
|
notifications of events occuring as well as allow a entity to post said
|
|
notifications to subscribers.
|
|
"""
|
|
|
|
RESERVED_KEYS = ('details',)
|
|
ANY = '*'
|
|
|
|
def __init__(self):
|
|
self._listeners = collections.defaultdict(list)
|
|
|
|
def reset(self):
|
|
self._listeners = collections.defaultdict(list)
|
|
|
|
def notify(self, state, details):
|
|
listeners = list(self._listeners.get(self.ANY, []))
|
|
for i in self._listeners[state]:
|
|
if i not in listeners:
|
|
listeners.append(i)
|
|
if not listeners:
|
|
return
|
|
for (callback, args, kwargs) in listeners:
|
|
if args is None:
|
|
args = []
|
|
if kwargs is None:
|
|
kwargs = {}
|
|
kwargs['details'] = details
|
|
try:
|
|
callback(state, *args, **kwargs)
|
|
except Exception:
|
|
LOG.exception(("Failure calling callback %s to notify about"
|
|
" state transition %s"), callback, state)
|
|
|
|
def register(self, state, callback, args=None, kwargs=None):
|
|
assert isinstance(callback, collections.Callable)
|
|
for i, (cb, args, kwargs) in enumerate(self._listeners.get(state, [])):
|
|
if cb is callback:
|
|
raise ValueError("Callback %s already registered" % (callback))
|
|
if kwargs:
|
|
for k in self.RESERVED_KEYS:
|
|
if k in kwargs:
|
|
raise KeyError(("Reserved key '%s' not allowed in "
|
|
"kwargs") % k)
|
|
kwargs = copy.copy(kwargs)
|
|
if args:
|
|
args = copy.copy(args)
|
|
self._listeners[state].append((callback, args, kwargs))
|
|
|
|
def deregister(self, state, callback):
|
|
if state not in self._listeners:
|
|
return
|
|
for i, (cb, args, kwargs) in enumerate(self._listeners[state]):
|
|
if cb is callback:
|
|
self._listeners[state].pop(i)
|
|
break
|
|
|
|
|
|
def copy_exc_info(exc_info):
|
|
"""Make copy of exception info tuple, as deep as possible"""
|
|
if exc_info is None:
|
|
return None
|
|
exc_type, exc_value, tb = exc_info
|
|
# NOTE(imelnikov): there is no need to copy type, and
|
|
# we can't copy traceback
|
|
return (exc_type, copy.deepcopy(exc_value), tb)
|
|
|
|
|
|
def are_equal_exc_info_tuples(ei1, ei2):
|
|
if ei1 == ei2:
|
|
return True
|
|
if ei1 is None or ei2 is None:
|
|
return False # if both are None, we returned True above
|
|
|
|
# NOTE(imelnikov): we can't compare exceptions with '=='
|
|
# because we want exc_info be equal to it's copy made with
|
|
# copy_exc_info above
|
|
if ei1[0] is not ei2[0]:
|
|
return False
|
|
if not all((type(ei1[1]) == type(ei2[1]),
|
|
str(ei1[1]) == str(ei2[1]),
|
|
repr(ei1[1]) == repr(ei2[1]))):
|
|
return False
|
|
if ei1[2] == ei2[2]:
|
|
return True
|
|
tb1 = traceback.format_tb(ei1[2])
|
|
tb2 = traceback.format_tb(ei2[2])
|
|
return tb1 == tb2
|
|
|
|
|
|
class Failure(object):
|
|
"""Object that represents failure.
|
|
|
|
Failure objects encapsulate exception information so that
|
|
it can be re-used later to re-raise or inspect.
|
|
"""
|
|
|
|
def __init__(self, exc_info=None, **kwargs):
|
|
if not kwargs:
|
|
if exc_info is None:
|
|
exc_info = sys.exc_info()
|
|
self._exc_info = exc_info
|
|
self._exc_type_names = list(
|
|
reflection.get_all_class_names(exc_info[0], up_to=Exception))
|
|
if not self._exc_type_names:
|
|
raise TypeError('Invalid exception type: %r' % exc_info[0])
|
|
self._exception_str = str(self._exc_info[1])
|
|
self._traceback_str = ''.join(
|
|
traceback.format_tb(self._exc_info[2]))
|
|
else:
|
|
self._exc_info = exc_info # may be None
|
|
self._exception_str = kwargs.pop('exception_str')
|
|
self._exc_type_names = kwargs.pop('exc_type_names', [])
|
|
self._traceback_str = kwargs.pop('traceback_str', None)
|
|
if kwargs:
|
|
raise TypeError(
|
|
'Failure.__init__ got unexpected keyword argument(s): %s'
|
|
% ', '.join(six.iterkeys(kwargs)))
|
|
|
|
def _matches(self, other):
|
|
if self is other:
|
|
return True
|
|
return (self._exc_type_names == other._exc_type_names
|
|
and self.exception_str == other.exception_str
|
|
and self.traceback_str == other.traceback_str)
|
|
|
|
def matches(self, other):
|
|
if not isinstance(other, Failure):
|
|
return False
|
|
if self.exc_info is None or other.exc_info is None:
|
|
return self._matches(other)
|
|
else:
|
|
return self == other
|
|
|
|
def __eq__(self, other):
|
|
if not isinstance(other, Failure):
|
|
return NotImplemented
|
|
return (self._matches(other) and
|
|
are_equal_exc_info_tuples(self.exc_info, other.exc_info))
|
|
|
|
def __ne__(self, other):
|
|
return not (self == other)
|
|
|
|
# NOTE(imelnikov): obj.__hash__() should return same values for equal
|
|
# objects, so we should redefine __hash__. Failure equality semantics
|
|
# is a bit complicated, so for now we just mark Failure objects as
|
|
# unhashable. See python docs on object.__hash__ for more info:
|
|
# http://docs.python.org/2/reference/datamodel.html#object.__hash__
|
|
__hash__ = None
|
|
|
|
@property
|
|
def exception(self):
|
|
"""Exception value, or None if exception value is not present.
|
|
|
|
Exception value may be lost during serialization.
|
|
"""
|
|
if self._exc_info:
|
|
return self._exc_info[1]
|
|
else:
|
|
return None
|
|
|
|
@property
|
|
def exception_str(self):
|
|
"""String representation of exception."""
|
|
return self._exception_str
|
|
|
|
@property
|
|
def exc_info(self):
|
|
"""Exception info tuple or None."""
|
|
return self._exc_info
|
|
|
|
@property
|
|
def traceback_str(self):
|
|
"""Exception traceback as string."""
|
|
return self._traceback_str
|
|
|
|
@staticmethod
|
|
def reraise_if_any(failures):
|
|
"""Re-raise exceptions if argument is not empty.
|
|
|
|
If argument is empty list, this method returns None. If
|
|
argument is list with single Failure object in it,
|
|
this failure is reraised. Else, WrappedFailure exception
|
|
is raised with failures list as causes.
|
|
"""
|
|
failures = list(failures)
|
|
if len(failures) == 1:
|
|
failures[0].reraise()
|
|
elif len(failures) > 1:
|
|
raise exceptions.WrappedFailure(failures)
|
|
|
|
def reraise(self):
|
|
"""Re-raise captured exception"""
|
|
if self._exc_info:
|
|
six.reraise(*self._exc_info)
|
|
else:
|
|
raise exceptions.WrappedFailure([self])
|
|
|
|
def check(self, *exc_classes):
|
|
"""Check if any of exc_classes caused the failure
|
|
|
|
Arguments of this method can be exception types or type
|
|
names (stings). If captured excption is instance of
|
|
exception of given type, the corresponding argument is
|
|
returned. Else, None is returned.
|
|
"""
|
|
for cls in exc_classes:
|
|
if isinstance(cls, type):
|
|
err = reflection.get_class_name(cls)
|
|
else:
|
|
err = cls
|
|
if err in self._exc_type_names:
|
|
return cls
|
|
return None
|
|
|
|
def __str__(self):
|
|
return 'Failure: %s: %s' % (self._exc_type_names[0],
|
|
self._exception_str)
|
|
|
|
def __iter__(self):
|
|
"""Iterate over exception type names"""
|
|
for et in self._exc_type_names:
|
|
yield et
|
|
|
|
def copy(self):
|
|
return Failure(exc_info=copy_exc_info(self.exc_info),
|
|
exception_str=self.exception_str,
|
|
traceback_str=self.traceback_str,
|
|
exc_type_names=self._exc_type_names[:])
|