openstack/deb-python-taskflow
Code Issues Proposed changes

Merge "Move failure to its own type specific module"

Browse Source
This commit is contained in:
Jenkins
2014-10-18 21:40:55 +00:00
committed by Gerrit Code Review
parent 371068dd1f f2ea4f1288
commit ab9b281213
7 changed files with 494 additions and 345 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
doc/source/types.rst
View File

@@ -10,7 +10,7 @@ Cache
Failure
=======
.. autoclass:: taskflow.utils.misc.Failure
.. automodule:: taskflow.types.failure
FSM
===

106
taskflow/tests/unit/test_utils_failure.py → taskflow/tests/unit/test_failure.py
View File

@@ -14,19 +14,29 @@
# License for the specific language governing permissions and limitations
# under the License.
import sys
import six
from taskflow import exceptions
from taskflow import test
from taskflow.tests import utils as test_utils
from taskflow.types import failure
from taskflow.utils import misc
def _captured_failure(msg):
try:
raise RuntimeError(msg)
except Exception:
return misc.Failure()
try:
raise RuntimeError(msg)
except Exception:
return failure.Failure()
def _make_exc_info(msg):
try:
raise RuntimeError(msg)
except Exception:
return sys.exc_info()
class GeneralFailureObjTestsMixin(object):
@@ -85,9 +95,9 @@ class ReCreatedFailureTestCase(test.TestCase, GeneralFailureObjTestsMixin):
def setUp(self):
super(ReCreatedFailureTestCase, self).setUp()
fail_obj = _captured_failure('Woot!')
self.fail_obj = misc.Failure(exception_str=fail_obj.exception_str,
traceback_str=fail_obj.traceback_str,
exc_type_names=list(fail_obj))
self.fail_obj = failure.Failure(exception_str=fail_obj.exception_str,
traceback_str=fail_obj.traceback_str,
exc_type_names=list(fail_obj))
def test_value_lost(self):
self.assertIs(self.fail_obj.exception, None)
@@ -109,7 +119,7 @@ class FromExceptionTestCase(test.TestCase, GeneralFailureObjTestsMixin):
def setUp(self):
super(FromExceptionTestCase, self).setUp()
self.fail_obj = misc.Failure.from_exception(RuntimeError('Woot!'))
self.fail_obj = failure.Failure.from_exception(RuntimeError('Woot!'))
def test_pformat_no_traceback(self):
text = self.fail_obj.pformat(traceback=True)
@@ -122,10 +132,10 @@ class FailureObjectTestCase(test.TestCase):
try:
raise SystemExit()
except BaseException:
self.assertRaises(TypeError, misc.Failure)
self.assertRaises(TypeError, failure.Failure)
def test_unknown_argument(self):
exc = self.assertRaises(TypeError, misc.Failure,
exc = self.assertRaises(TypeError, failure.Failure,
exception_str='Woot!',
traceback_str=None,
exc_type_names=['Exception'],
@@ -134,12 +144,12 @@ class FailureObjectTestCase(test.TestCase):
self.assertEqual(str(exc), expected)
def test_empty_does_not_reraise(self):
self.assertIs(misc.Failure.reraise_if_any([]), None)
self.assertIs(failure.Failure.reraise_if_any([]), None)
def test_reraises_one(self):
fls = [_captured_failure('Woot!')]
self.assertRaisesRegexp(RuntimeError, '^Woot!$',
misc.Failure.reraise_if_any, fls)
failure.Failure.reraise_if_any, fls)
def test_reraises_several(self):
fls = [
@@ -147,7 +157,7 @@ class FailureObjectTestCase(test.TestCase):
_captured_failure('Oh, not again!')
]
exc = self.assertRaises(exceptions.WrappedFailure,
misc.Failure.reraise_if_any, fls)
failure.Failure.reraise_if_any, fls)
self.assertEqual(list(exc), fls)
def test_failure_copy(self):
@@ -160,9 +170,9 @@ class FailureObjectTestCase(test.TestCase):
def test_failure_copy_recaptured(self):
captured = _captured_failure('Woot!')
fail_obj = misc.Failure(exception_str=captured.exception_str,
traceback_str=captured.traceback_str,
exc_type_names=list(captured))
fail_obj = failure.Failure(exception_str=captured.exception_str,
traceback_str=captured.traceback_str,
exc_type_names=list(captured))
copied = fail_obj.copy()
self.assertIsNot(fail_obj, copied)
self.assertEqual(fail_obj, copied)
@@ -171,9 +181,9 @@ class FailureObjectTestCase(test.TestCase):
def test_recaptured_not_eq(self):
captured = _captured_failure('Woot!')
fail_obj = misc.Failure(exception_str=captured.exception_str,
traceback_str=captured.traceback_str,
exc_type_names=list(captured))
fail_obj = failure.Failure(exception_str=captured.exception_str,
traceback_str=captured.traceback_str,
exc_type_names=list(captured))
self.assertFalse(fail_obj == captured)
self.assertTrue(fail_obj != captured)
self.assertTrue(fail_obj.matches(captured))
@@ -185,13 +195,13 @@ class FailureObjectTestCase(test.TestCase):
def test_two_recaptured_neq(self):
captured = _captured_failure('Woot!')
fail_obj = misc.Failure(exception_str=captured.exception_str,
traceback_str=captured.traceback_str,
exc_type_names=list(captured))
fail_obj = failure.Failure(exception_str=captured.exception_str,
traceback_str=captured.traceback_str,
exc_type_names=list(captured))
new_exc_str = captured.exception_str.replace('Woot', 'w00t')
fail_obj2 = misc.Failure(exception_str=new_exc_str,
traceback_str=captured.traceback_str,
exc_type_names=list(captured))
fail_obj2 = failure.Failure(exception_str=new_exc_str,
traceback_str=captured.traceback_str,
exc_type_names=list(captured))
self.assertNotEqual(fail_obj, fail_obj2)
self.assertFalse(fail_obj2.matches(fail_obj))
@@ -242,7 +252,7 @@ class WrappedFailureTestCase(test.TestCase):
try:
raise exceptions.WrappedFailure([f1, f2])
except Exception:
fail_obj = misc.Failure()
fail_obj = failure.Failure()
wf = exceptions.WrappedFailure([fail_obj, f3])
self.assertEqual(list(wf), [f1, f2, f3])
@@ -252,13 +262,13 @@ class NonAsciiExceptionsTestCase(test.TestCase):
def test_exception_with_non_ascii_str(self):
bad_string = chr(200)
fail = misc.Failure.from_exception(ValueError(bad_string))
fail = failure.Failure.from_exception(ValueError(bad_string))
self.assertEqual(fail.exception_str, bad_string)
self.assertEqual(str(fail), 'Failure: ValueError: %s' % bad_string)
def test_exception_non_ascii_unicode(self):
hi_ru = u'привет'
fail = misc.Failure.from_exception(ValueError(hi_ru))
fail = failure.Failure.from_exception(ValueError(hi_ru))
self.assertEqual(fail.exception_str, hi_ru)
self.assertIsInstance(fail.exception_str, six.text_type)
self.assertEqual(six.text_type(fail),
@@ -268,7 +278,7 @@ class NonAsciiExceptionsTestCase(test.TestCase):
hi_cn = u'嗨'
fail = ValueError(hi_cn)
self.assertEqual(hi_cn, exceptions.exception_message(fail))
fail = misc.Failure.from_exception(fail)
fail = failure.Failure.from_exception(fail)
wrapped_fail = exceptions.WrappedFailure([fail])
if six.PY2:
# Python 2.x will unicode escape it, while python 3.3+ will not,
@@ -283,12 +293,46 @@ class NonAsciiExceptionsTestCase(test.TestCase):
def test_failure_equality_with_non_ascii_str(self):
bad_string = chr(200)
fail = misc.Failure.from_exception(ValueError(bad_string))
fail = failure.Failure.from_exception(ValueError(bad_string))
copied = fail.copy()
self.assertEqual(fail, copied)
def test_failure_equality_non_ascii_unicode(self):
hi_ru = u'привет'
fail = misc.Failure.from_exception(ValueError(hi_ru))
fail = failure.Failure.from_exception(ValueError(hi_ru))
copied = fail.copy()
self.assertEqual(fail, copied)
class ExcInfoUtilsTest(test.TestCase):
def test_copy_none(self):
result = failure._copy_exc_info(None)
self.assertIsNone(result)
def test_copy_exc_info(self):
exc_info = _make_exc_info("Woot!")
result = failure._copy_exc_info(exc_info)
self.assertIsNot(result, exc_info)
self.assertIs(result[0], RuntimeError)
self.assertIsNot(result[1], exc_info[1])
self.assertIs(result[2], exc_info[2])
def test_none_equals(self):
self.assertTrue(failure._are_equal_exc_info_tuples(None, None))
def test_none_ne_tuple(self):
exc_info = _make_exc_info("Woot!")
self.assertFalse(failure._are_equal_exc_info_tuples(None, exc_info))
def test_tuple_nen_none(self):
exc_info = _make_exc_info("Woot!")
self.assertFalse(failure._are_equal_exc_info_tuples(exc_info, None))
def test_tuple_equals_itself(self):
exc_info = _make_exc_info("Woot!")
self.assertTrue(failure._are_equal_exc_info_tuples(exc_info, exc_info))
def test_typle_equals_copy(self):
exc_info = _make_exc_info("Woot!")
copied = failure._copy_exc_info(exc_info)
self.assertTrue(failure._are_equal_exc_info_tuples(exc_info, copied))

8
taskflow/tests/unit/test_retries.py
View File

@@ -23,7 +23,7 @@ from taskflow import retry
from taskflow import states as st
from taskflow import test
from taskflow.tests import utils
from taskflow.utils import misc
from taskflow.types import failure
class RetryTest(utils.EngineTestBase):
@@ -559,7 +559,7 @@ class RetryTest(utils.EngineTestBase):
# we execute retry
engine.storage.save('flow-1_retry', 1)
# task fails
fail = misc.Failure.from_exception(RuntimeError('foo')),
fail = failure.Failure.from_exception(RuntimeError('foo')),
engine.storage.save('task1', fail, state=st.FAILURE)
if when == 'task fails':
return engine
@@ -635,7 +635,7 @@ class RetryTest(utils.EngineTestBase):
self._make_engine(flow).run)
self.assertEqual(len(r.history), 1)
self.assertEqual(r.history[0][1], {})
self.assertEqual(isinstance(r.history[0][0], misc.Failure), True)
self.assertEqual(isinstance(r.history[0][0], failure.Failure), True)
def test_retry_revert_fails(self):
@@ -693,7 +693,7 @@ class RetryTest(utils.EngineTestBase):
engine.storage.save('test2_retry', 1)
engine.storage.save('b', 11)
# pretend that 'c' failed
fail = misc.Failure.from_exception(RuntimeError('Woot!'))
fail = failure.Failure.from_exception(RuntimeError('Woot!'))
engine.storage.save('c', fail, st.FAILURE)
engine.run()

47
taskflow/tests/unit/test_utils.py
View File

@@ -18,7 +18,6 @@ import collections
import functools
import inspect
import random
import sys
import threading
import time
@@ -28,6 +27,7 @@ import testtools
from taskflow import states
from taskflow import test
from taskflow.tests import utils as test_utils
from taskflow.types import failure
from taskflow.utils import lock_utils
from taskflow.utils import misc
from taskflow.utils import reflection
@@ -335,8 +335,8 @@ class GetClassNameTest(test.TestCase):
self.assertEqual(name, 'RuntimeError')
def test_global_class(self):
name = reflection.get_class_name(misc.Failure)
self.assertEqual(name, 'taskflow.utils.misc.Failure')
name = reflection.get_class_name(failure.Failure)
self.assertEqual(name, 'taskflow.types.failure.Failure')
def test_class(self):
name = reflection.get_class_name(Class)
@@ -619,47 +619,6 @@ class UriParseTest(test.TestCase):
self.assertEqual(None, parsed.password)
class ExcInfoUtilsTest(test.TestCase):
def _make_ex_info(self):
try:
raise RuntimeError('Woot!')
except Exception:
return sys.exc_info()
def test_copy_none(self):
result = misc.copy_exc_info(None)
self.assertIsNone(result)
def test_copy_exc_info(self):
exc_info = self._make_ex_info()
result = misc.copy_exc_info(exc_info)
self.assertIsNot(result, exc_info)
self.assertIs(result[0], RuntimeError)
self.assertIsNot(result[1], exc_info[1])
self.assertIs(result[2], exc_info[2])
def test_none_equals(self):
self.assertTrue(misc.are_equal_exc_info_tuples(None, None))
def test_none_ne_tuple(self):
exc_info = self._make_ex_info()
self.assertFalse(misc.are_equal_exc_info_tuples(None, exc_info))
def test_tuple_nen_none(self):
exc_info = self._make_ex_info()
self.assertFalse(misc.are_equal_exc_info_tuples(exc_info, None))
def test_tuple_equals_itself(self):
exc_info = self._make_ex_info()
self.assertTrue(misc.are_equal_exc_info_tuples(exc_info, exc_info))
def test_typle_equals_copy(self):
exc_info = self._make_ex_info()
copied = misc.copy_exc_info(exc_info)
self.assertTrue(misc.are_equal_exc_info_tuples(exc_info, copied))
class TestSequenceMinus(test.TestCase):
def test_simple_case(self):

287
taskflow/types/failure.py Normal file
View File

@@ -0,0 +1,287 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import copy
import sys
import traceback
import six
from taskflow import exceptions as exc
from taskflow.utils import reflection
def _copy_exc_info(exc_info):
"""Make copy of exception info tuple, as deep as possible."""
if exc_info is None:
return None
exc_type, exc_value, tb = exc_info
# NOTE(imelnikov): there is no need to copy type, and
# we can't copy traceback.
return (exc_type, copy.deepcopy(exc_value), tb)
def _are_equal_exc_info_tuples(ei1, ei2):
if ei1 == ei2:
return True
if ei1 is None or ei2 is None:
return False # if both are None, we returned True above
# NOTE(imelnikov): we can't compare exceptions with '=='
# because we want exc_info be equal to it's copy made with
# copy_exc_info above.
if ei1[0] is not ei2[0]:
return False
if not all((type(ei1[1]) == type(ei2[1]),
exc.exception_message(ei1[1]) == exc.exception_message(ei2[1]),
repr(ei1[1]) == repr(ei2[1]))):
return False
if ei1[2] == ei2[2]:
return True
tb1 = traceback.format_tb(ei1[2])
tb2 = traceback.format_tb(ei2[2])
return tb1 == tb2
class Failure(object):
"""Object that represents failure.
Failure objects encapsulate exception information so that they can be
re-used later to re-raise, inspect, examine, log, print, serialize,
deserialize...
One example where they are dependened upon is in the WBE engine. When a
remote worker throws an exception, the WBE based engine will receive that
exception and desire to reraise it to the user/caller of the WBE based
engine for appropriate handling (this matches the behavior of non-remote
engines). To accomplish this a failure object (or a
:py:meth:`~misc.Failure.to_dict` form) would be sent over the WBE channel
and the WBE based engine would deserialize it and use this objects
:meth:`.reraise` method to cause an exception that contains
similar/equivalent information as the original exception to be reraised,
allowing the user (or the WBE engine itself) to then handle the worker
failure/exception as they desire.
For those who are curious, here are a few reasons why the original
exception itself *may* not be reraised and instead a reraised wrapped
failure exception object will be instead. These explanations are *only*
applicable when a failure object is serialized and deserialized (when it is
retained inside the python process that the exception was created in the
the original exception can be reraised correctly without issue).
* Traceback objects are not serializable/recreatable, since they contain
references to stack frames at the location where the exception was
raised. When a failure object is serialized and sent across a channel
and recreated it is *not* possible to restore the original traceback and
originating stack frames.
* The original exception *type* can not be guaranteed to be found, workers
can run code that is not accessible/available when the failure is being
deserialized. Even if it was possible to use pickle safely it would not
be possible to find the originating exception or associated code in this
situation.
* The original exception *type* can not be guaranteed to be constructed in
a *correct* manner. At the time of failure object creation the exception
has already been created and the failure object can not assume it has
knowledge (or the ability) to recreate the original type of the captured
exception (this is especially hard if the original exception was created
via a complex process via some custom exception constructor).
* The original exception *type* can not be guaranteed to be constructed in
a *safe* manner. Importing *foreign* exception types dynamically can be
problematic when not done correctly and in a safe manner; since failure
objects can capture any exception it would be *unsafe* to try to import
those exception types namespaces and modules on the receiver side
dynamically (this would create similar issues as the ``pickle`` module in
python has where foreign modules can be imported, causing those modules
to have code ran when this happens, and this can cause issues and
side-effects that the receiver would not have intended to have caused).
"""
DICT_VERSION = 1
def __init__(self, exc_info=None, **kwargs):
if not kwargs:
if exc_info is None:
exc_info = sys.exc_info()
self._exc_info = exc_info
self._exc_type_names = list(
reflection.get_all_class_names(exc_info[0], up_to=Exception))
if not self._exc_type_names:
raise TypeError('Invalid exception type: %r' % exc_info[0])
self._exception_str = exc.exception_message(self._exc_info[1])
self._traceback_str = ''.join(
traceback.format_tb(self._exc_info[2]))
else:
self._exc_info = exc_info # may be None
self._exception_str = kwargs.pop('exception_str')
self._exc_type_names = kwargs.pop('exc_type_names', [])
self._traceback_str = kwargs.pop('traceback_str', None)
if kwargs:
raise TypeError(
'Failure.__init__ got unexpected keyword argument(s): %s'
% ', '.join(six.iterkeys(kwargs)))
@classmethod
def from_exception(cls, exception):
"""Creates a failure object from a exception instance."""
return cls((type(exception), exception, None))
def _matches(self, other):
if self is other:
return True
return (self._exc_type_names == other._exc_type_names
and self.exception_str == other.exception_str
and self.traceback_str == other.traceback_str)
def matches(self, other):
"""Checks if another object is equivalent to this object."""
if not isinstance(other, Failure):
return False
if self.exc_info is None or other.exc_info is None:
return self._matches(other)
else:
return self == other
def __eq__(self, other):
if not isinstance(other, Failure):
return NotImplemented
return (self._matches(other) and
_are_equal_exc_info_tuples(self.exc_info, other.exc_info))
def __ne__(self, other):
return not (self == other)
# NOTE(imelnikov): obj.__hash__() should return same values for equal
# objects, so we should redefine __hash__. Failure equality semantics
# is a bit complicated, so for now we just mark Failure objects as
# unhashable. See python docs on object.__hash__ for more info:
# http://docs.python.org/2/reference/datamodel.html#object.__hash__
__hash__ = None
@property
def exception(self):
"""Exception value, or None if exception value is not present.
Exception value may be lost during serialization.
"""
if self._exc_info:
return self._exc_info[1]
else:
return None
@property
def exception_str(self):
"""String representation of exception."""
return self._exception_str
@property
def exc_info(self):
"""Exception info tuple or None."""
return self._exc_info
@property
def traceback_str(self):
"""Exception traceback as string."""
return self._traceback_str
@staticmethod
def reraise_if_any(failures):
"""Re-raise exceptions if argument is not empty.
If argument is empty list, this method returns None. If
argument is a list with a single ``Failure`` object in it,
that failure is reraised. Else, a
:class:`~taskflow.exceptions.WrappedFailure` exception
is raised with a failure list as causes.
"""
failures = list(failures)
if len(failures) == 1:
failures[0].reraise()
elif len(failures) > 1:
raise exc.WrappedFailure(failures)
def reraise(self):
"""Re-raise captured exception."""
if self._exc_info:
six.reraise(*self._exc_info)
else:
raise exc.WrappedFailure([self])
def check(self, *exc_classes):
"""Check if any of ``exc_classes`` caused the failure.
Arguments of this method can be exception types or type
names (stings). If captured exception is instance of
exception of given type, the corresponding argument is
returned. Else, None is returned.
"""
for cls in exc_classes:
if isinstance(cls, type):
err = reflection.get_class_name(cls)
else:
err = cls
if err in self._exc_type_names:
return cls
return None
def __str__(self):
return self.pformat()
def pformat(self, traceback=False):
"""Pretty formats the failure object into a string."""
buf = six.StringIO()
buf.write(
'Failure: %s: %s' % (self._exc_type_names[0], self._exception_str))
if traceback:
if self._traceback_str is not None:
traceback_str = self._traceback_str.rstrip()
else:
traceback_str = None
if traceback_str:
buf.write('\nTraceback (most recent call last):\n')
buf.write(traceback_str)
else:
buf.write('\nTraceback not available.')
return buf.getvalue()
def __iter__(self):
"""Iterate over exception type names."""
for et in self._exc_type_names:
yield et
@classmethod
def from_dict(cls, data):
"""Converts this from a dictionary to a object."""
data = dict(data)
version = data.pop('version', None)
if version != cls.DICT_VERSION:
raise ValueError('Invalid dict version of failure object: %r'
% version)
return cls(**data)
def to_dict(self):
"""Converts this object to a dictionary."""
return {
'exception_str': self.exception_str,
'traceback_str': self.traceback_str,
'exc_type_names': list(self),
'version': self.DICT_VERSION,
}
def copy(self):
"""Copies this object."""
return Failure(exc_info=_copy_exc_info(self.exc_info),
exception_str=self.exception_str,
traceback_str=self.traceback_str,
exc_type_names=self._exc_type_names[:])

118
taskflow/utils/deprecation.py Normal file
View File

@@ -0,0 +1,118 @@
# -*- coding: utf-8 -*-
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import warnings
from taskflow.utils import reflection
def deprecation(message, stacklevel=2):
"""Warns about some type of deprecation that has been made."""
warnings.warn(message, category=DeprecationWarning, stacklevel=stacklevel)
# Helper accessors for the moved proxy (since it will not have easy access
# to its own getattr and setattr functions).
_setattr = object.__setattr__
_getattr = object.__getattribute__
class MovedClassProxy(object):
"""Acts as a proxy to a class that was moved to another location.
Partially based on:
http://code.activestate.com/recipes/496741-object-proxying/ and other
various examination of how to make a good enough proxy for our usage to
move the various types we want to move during the deprecation process.
And partially based on the wrapt object proxy (which we should just use
when it becomes available @ http://review.openstack.org/#/c/94754/).
"""
__slots__ = [
'__wrapped__', '__message__', '__stacklevel__',
# Ensure weakrefs can be made,
# https://docs.python.org/2/reference/datamodel.html#slots
'__weakref__',
]
def __init__(self, wrapped, message, stacklevel):
# We can't assign to these directly, since we are overriding getattr
# and setattr and delattr so we have to do this hoop jump to ensure
# that we don't invoke those methods (and cause infinite recursion).
_setattr(self, '__wrapped__', wrapped)
_setattr(self, '__message__', message)
_setattr(self, '__stacklevel__', stacklevel)
try:
_setattr(self, '__qualname__', wrapped.__qualname__)
except AttributeError:
pass
def __instancecheck__(self, instance):
deprecation(
_getattr(self, '__message__'), _getattr(self, '__stacklevel__'))
return isinstance(instance, _getattr(self, '__wrapped__'))
def __subclasscheck__(self, instance):
deprecation(
_getattr(self, '__message__'), _getattr(self, '__stacklevel__'))
return issubclass(instance, _getattr(self, '__wrapped__'))
def __call__(self, *args, **kwargs):
deprecation(
_getattr(self, '__message__'), _getattr(self, '__stacklevel__'))
return _getattr(self, '__wrapped__')(*args, **kwargs)
def __getattribute__(self, name):
return getattr(_getattr(self, '__wrapped__'), name)
def __setattr__(self, name, value):
setattr(_getattr(self, '__wrapped__'), name, value)
def __delattr__(self, name):
delattr(_getattr(self, '__wrapped__'), name)
def __repr__(self):
wrapped = _getattr(self, '__wrapped__')
return "<%s at 0x%x for %r at 0x%x>" % (
type(self).__name__, id(self), wrapped, id(wrapped))
def moved_class(new_class, old_class_name, old_module_name, message=None,
version=None, removal_version=None):
"""Deprecates a class that was moved to another location.
This will emit warnings when the old locations class is initialized,
telling where the new and improved location for the old class now is.
"""
old_name = ".".join((old_module_name, old_class_name))
new_name = reflection.get_class_name(new_class)
message_components = [
"Class '%s' has moved to '%s'" % (old_name, new_name),
]
if version:
message_components.append(" in version '%s'" % version)
if removal_version:
if removal_version == "?":
message_components.append(" and will be removed in a future"
" version")
else:
message_components.append(" and will be removed in version '%s'"
% removal_version)
if message:
message_components.append(": %s" % message)
return MovedClassProxy(new_class, "".join(message_components), 3)

271
taskflow/utils/misc.py
View File

@@ -28,7 +28,6 @@ import re
import string
import sys
import threading
import traceback
from oslo.serialization import jsonutils
from oslo.utils import netutils
@@ -37,7 +36,8 @@ from six.moves import map as compat_map
from six.moves import range as compat_range
from six.moves.urllib import parse as urlparse
from taskflow import exceptions as exc
from taskflow.types import failure
from taskflow.utils import deprecation
from taskflow.utils import reflection
@@ -392,6 +392,10 @@ def ensure_tree(path):
raise
Failure = deprecation.moved_class(failure.Failure, 'Failure', __name__,
version="0.5", removal_version="?")
class Notifier(object):
"""A notification helper class.
@@ -489,38 +493,6 @@ class Notifier(object):
break
def copy_exc_info(exc_info):
"""Make copy of exception info tuple, as deep as possible."""
if exc_info is None:
return None
exc_type, exc_value, tb = exc_info
# NOTE(imelnikov): there is no need to copy type, and
# we can't copy traceback.
return (exc_type, copy.deepcopy(exc_value), tb)
def are_equal_exc_info_tuples(ei1, ei2):
if ei1 == ei2:
return True
if ei1 is None or ei2 is None:
return False # if both are None, we returned True above
# NOTE(imelnikov): we can't compare exceptions with '=='
# because we want exc_info be equal to it's copy made with
# copy_exc_info above.
if ei1[0] is not ei2[0]:
return False
if not all((type(ei1[1]) == type(ei2[1]),
exc.exception_message(ei1[1]) == exc.exception_message(ei2[1]),
repr(ei1[1]) == repr(ei2[1]))):
return False
if ei1[2] == ei2[2]:
return True
tb1 = traceback.format_tb(ei1[2])
tb2 = traceback.format_tb(ei2[2])
return tb1 == tb2
@contextlib.contextmanager
def capture_failure():
"""Captures the occurring exception and provides a failure object back.
@@ -551,234 +523,3 @@ def capture_failure():
raise RuntimeError("No active exception is being handled")
else:
yield Failure(exc_info=exc_info)
class Failure(object):
"""Object that represents failure.
Failure objects encapsulate exception information so that they can be
re-used later to re-raise, inspect, examine, log, print, serialize,
deserialize...
One example where they are dependened upon is in the WBE engine. When a
remote worker throws an exception, the WBE based engine will receive that
exception and desire to reraise it to the user/caller of the WBE based
engine for appropriate handling (this matches the behavior of non-remote
engines). To accomplish this a failure object (or a
:py:meth:`~misc.Failure.to_dict` form) would be sent over the WBE channel
and the WBE based engine would deserialize it and use this objects
:meth:`.reraise` method to cause an exception that contains
similar/equivalent information as the original exception to be reraised,
allowing the user (or the WBE engine itself) to then handle the worker
failure/exception as they desire.
For those who are curious, here are a few reasons why the original
exception itself *may* not be reraised and instead a reraised wrapped
failure exception object will be instead. These explanations are *only*
applicable when a failure object is serialized and deserialized (when it is
retained inside the python process that the exception was created in the
the original exception can be reraised correctly without issue).
* Traceback objects are not serializable/recreatable, since they contain
references to stack frames at the location where the exception was
raised. When a failure object is serialized and sent across a channel
and recreated it is *not* possible to restore the original traceback and
originating stack frames.
* The original exception *type* can not be guaranteed to be found, workers
can run code that is not accessible/available when the failure is being
deserialized. Even if it was possible to use pickle safely it would not
be possible to find the originating exception or associated code in this
situation.
* The original exception *type* can not be guaranteed to be constructed in
a *correct* manner. At the time of failure object creation the exception
has already been created and the failure object can not assume it has
knowledge (or the ability) to recreate the original type of the captured
exception (this is especially hard if the original exception was created
via a complex process via some custom exception constructor).
* The original exception *type* can not be guaranteed to be constructed in
a *safe* manner. Importing *foreign* exception types dynamically can be
problematic when not done correctly and in a safe manner; since failure
objects can capture any exception it would be *unsafe* to try to import
those exception types namespaces and modules on the receiver side
dynamically (this would create similar issues as the ``pickle`` module in
python has where foreign modules can be imported, causing those modules
to have code ran when this happens, and this can cause issues and
side-effects that the receiver would not have intended to have caused).
"""
DICT_VERSION = 1
def __init__(self, exc_info=None, **kwargs):
if not kwargs:
if exc_info is None:
exc_info = sys.exc_info()
self._exc_info = exc_info
self._exc_type_names = list(
reflection.get_all_class_names(exc_info[0], up_to=Exception))
if not self._exc_type_names:
raise TypeError('Invalid exception type: %r' % exc_info[0])
self._exception_str = exc.exception_message(self._exc_info[1])
self._traceback_str = ''.join(
traceback.format_tb(self._exc_info[2]))
else:
self._exc_info = exc_info # may be None
self._exception_str = kwargs.pop('exception_str')
self._exc_type_names = kwargs.pop('exc_type_names', [])
self._traceback_str = kwargs.pop('traceback_str', None)
if kwargs:
raise TypeError(
'Failure.__init__ got unexpected keyword argument(s): %s'
% ', '.join(six.iterkeys(kwargs)))
@classmethod
def from_exception(cls, exception):
"""Creates a failure object from a exception instance."""
return cls((type(exception), exception, None))
def _matches(self, other):
if self is other:
return True
return (self._exc_type_names == other._exc_type_names
and self.exception_str == other.exception_str
and self.traceback_str == other.traceback_str)
def matches(self, other):
"""Checks if another object is equivalent to this object."""
if not isinstance(other, Failure):
return False
if self.exc_info is None or other.exc_info is None:
return self._matches(other)
else:
return self == other
def __eq__(self, other):
if not isinstance(other, Failure):
return NotImplemented
return (self._matches(other) and
are_equal_exc_info_tuples(self.exc_info, other.exc_info))
def __ne__(self, other):
return not (self == other)
# NOTE(imelnikov): obj.__hash__() should return same values for equal
# objects, so we should redefine __hash__. Failure equality semantics
# is a bit complicated, so for now we just mark Failure objects as
# unhashable. See python docs on object.__hash__ for more info:
# http://docs.python.org/2/reference/datamodel.html#object.__hash__
__hash__ = None
@property
def exception(self):
"""Exception value, or None if exception value is not present.
Exception value may be lost during serialization.
"""
if self._exc_info:
return self._exc_info[1]
else:
return None
@property
def exception_str(self):
"""String representation of exception."""
return self._exception_str
@property
def exc_info(self):
"""Exception info tuple or None."""
return self._exc_info
@property
def traceback_str(self):
"""Exception traceback as string."""
return self._traceback_str
@staticmethod
def reraise_if_any(failures):
"""Re-raise exceptions if argument is not empty.
If argument is empty list, this method returns None. If
argument is a list with a single ``Failure`` object in it,
that failure is reraised. Else, a
:class:`~taskflow.exceptions.WrappedFailure` exception
is raised with a failure list as causes.
"""
failures = list(failures)
if len(failures) == 1:
failures[0].reraise()
elif len(failures) > 1:
raise exc.WrappedFailure(failures)
def reraise(self):
"""Re-raise captured exception."""
if self._exc_info:
six.reraise(*self._exc_info)
else:
raise exc.WrappedFailure([self])
def check(self, *exc_classes):
"""Check if any of ``exc_classes`` caused the failure.
Arguments of this method can be exception types or type
names (stings). If captured exception is instance of
exception of given type, the corresponding argument is
returned. Else, None is returned.
"""
for cls in exc_classes:
if isinstance(cls, type):
err = reflection.get_class_name(cls)
else:
err = cls
if err in self._exc_type_names:
return cls
return None
def __str__(self):
return self.pformat()
def pformat(self, traceback=False):
"""Pretty formats the failure object into a string."""
buf = six.StringIO()
buf.write(
'Failure: %s: %s' % (self._exc_type_names[0], self._exception_str))
if traceback:
if self._traceback_str is not None:
traceback_str = self._traceback_str.rstrip()
else:
traceback_str = None
if traceback_str:
buf.write('\nTraceback (most recent call last):\n')
buf.write(traceback_str)
else:
buf.write('\nTraceback not available.')
return buf.getvalue()
def __iter__(self):
"""Iterate over exception type names."""
for et in self._exc_type_names:
yield et
@classmethod
def from_dict(cls, data):
"""Converts this from a dictionary to a object."""
data = dict(data)
version = data.pop('version', None)
if version != cls.DICT_VERSION:
raise ValueError('Invalid dict version of failure object: %r'
% version)
return cls(**data)
def to_dict(self):
"""Converts this object to a dictionary."""
return {
'exception_str': self.exception_str,
'traceback_str': self.traceback_str,
'exc_type_names': list(self),
'version': self.DICT_VERSION,
}
def copy(self):
"""Copies this object."""
return Failure(exc_info=copy_exc_info(self.exc_info),
exception_str=self.exception_str,
traceback_str=self.traceback_str,
exc_type_names=self._exc_type_names[:])