diff --git a/doc/source/types.rst b/doc/source/types.rst index 1f573c8a..5c53db72 100644 --- a/doc/source/types.rst +++ b/doc/source/types.rst @@ -10,7 +10,7 @@ Cache Failure ======= -.. autoclass:: taskflow.utils.misc.Failure +.. automodule:: taskflow.types.failure FSM === diff --git a/taskflow/tests/unit/test_utils_failure.py b/taskflow/tests/unit/test_failure.py similarity index 73% rename from taskflow/tests/unit/test_utils_failure.py rename to taskflow/tests/unit/test_failure.py index 4958da62..3f4d001e 100644 --- a/taskflow/tests/unit/test_utils_failure.py +++ b/taskflow/tests/unit/test_failure.py @@ -14,19 +14,29 @@ # License for the specific language governing permissions and limitations # under the License. +import sys + import six from taskflow import exceptions from taskflow import test from taskflow.tests import utils as test_utils +from taskflow.types import failure from taskflow.utils import misc def _captured_failure(msg): - try: - raise RuntimeError(msg) - except Exception: - return misc.Failure() + try: + raise RuntimeError(msg) + except Exception: + return failure.Failure() + + +def _make_exc_info(msg): + try: + raise RuntimeError(msg) + except Exception: + return sys.exc_info() class GeneralFailureObjTestsMixin(object): @@ -85,9 +95,9 @@ class ReCreatedFailureTestCase(test.TestCase, GeneralFailureObjTestsMixin): def setUp(self): super(ReCreatedFailureTestCase, self).setUp() fail_obj = _captured_failure('Woot!') - self.fail_obj = misc.Failure(exception_str=fail_obj.exception_str, - traceback_str=fail_obj.traceback_str, - exc_type_names=list(fail_obj)) + self.fail_obj = failure.Failure(exception_str=fail_obj.exception_str, + traceback_str=fail_obj.traceback_str, + exc_type_names=list(fail_obj)) def test_value_lost(self): self.assertIs(self.fail_obj.exception, None) @@ -109,7 +119,7 @@ class FromExceptionTestCase(test.TestCase, GeneralFailureObjTestsMixin): def setUp(self): super(FromExceptionTestCase, self).setUp() - self.fail_obj = misc.Failure.from_exception(RuntimeError('Woot!')) + self.fail_obj = failure.Failure.from_exception(RuntimeError('Woot!')) def test_pformat_no_traceback(self): text = self.fail_obj.pformat(traceback=True) @@ -122,10 +132,10 @@ class FailureObjectTestCase(test.TestCase): try: raise SystemExit() except BaseException: - self.assertRaises(TypeError, misc.Failure) + self.assertRaises(TypeError, failure.Failure) def test_unknown_argument(self): - exc = self.assertRaises(TypeError, misc.Failure, + exc = self.assertRaises(TypeError, failure.Failure, exception_str='Woot!', traceback_str=None, exc_type_names=['Exception'], @@ -134,12 +144,12 @@ class FailureObjectTestCase(test.TestCase): self.assertEqual(str(exc), expected) def test_empty_does_not_reraise(self): - self.assertIs(misc.Failure.reraise_if_any([]), None) + self.assertIs(failure.Failure.reraise_if_any([]), None) def test_reraises_one(self): fls = [_captured_failure('Woot!')] self.assertRaisesRegexp(RuntimeError, '^Woot!$', - misc.Failure.reraise_if_any, fls) + failure.Failure.reraise_if_any, fls) def test_reraises_several(self): fls = [ @@ -147,7 +157,7 @@ class FailureObjectTestCase(test.TestCase): _captured_failure('Oh, not again!') ] exc = self.assertRaises(exceptions.WrappedFailure, - misc.Failure.reraise_if_any, fls) + failure.Failure.reraise_if_any, fls) self.assertEqual(list(exc), fls) def test_failure_copy(self): @@ -160,9 +170,9 @@ class FailureObjectTestCase(test.TestCase): def test_failure_copy_recaptured(self): captured = _captured_failure('Woot!') - fail_obj = misc.Failure(exception_str=captured.exception_str, - traceback_str=captured.traceback_str, - exc_type_names=list(captured)) + fail_obj = failure.Failure(exception_str=captured.exception_str, + traceback_str=captured.traceback_str, + exc_type_names=list(captured)) copied = fail_obj.copy() self.assertIsNot(fail_obj, copied) self.assertEqual(fail_obj, copied) @@ -171,9 +181,9 @@ class FailureObjectTestCase(test.TestCase): def test_recaptured_not_eq(self): captured = _captured_failure('Woot!') - fail_obj = misc.Failure(exception_str=captured.exception_str, - traceback_str=captured.traceback_str, - exc_type_names=list(captured)) + fail_obj = failure.Failure(exception_str=captured.exception_str, + traceback_str=captured.traceback_str, + exc_type_names=list(captured)) self.assertFalse(fail_obj == captured) self.assertTrue(fail_obj != captured) self.assertTrue(fail_obj.matches(captured)) @@ -185,13 +195,13 @@ class FailureObjectTestCase(test.TestCase): def test_two_recaptured_neq(self): captured = _captured_failure('Woot!') - fail_obj = misc.Failure(exception_str=captured.exception_str, - traceback_str=captured.traceback_str, - exc_type_names=list(captured)) + fail_obj = failure.Failure(exception_str=captured.exception_str, + traceback_str=captured.traceback_str, + exc_type_names=list(captured)) new_exc_str = captured.exception_str.replace('Woot', 'w00t') - fail_obj2 = misc.Failure(exception_str=new_exc_str, - traceback_str=captured.traceback_str, - exc_type_names=list(captured)) + fail_obj2 = failure.Failure(exception_str=new_exc_str, + traceback_str=captured.traceback_str, + exc_type_names=list(captured)) self.assertNotEqual(fail_obj, fail_obj2) self.assertFalse(fail_obj2.matches(fail_obj)) @@ -242,7 +252,7 @@ class WrappedFailureTestCase(test.TestCase): try: raise exceptions.WrappedFailure([f1, f2]) except Exception: - fail_obj = misc.Failure() + fail_obj = failure.Failure() wf = exceptions.WrappedFailure([fail_obj, f3]) self.assertEqual(list(wf), [f1, f2, f3]) @@ -252,13 +262,13 @@ class NonAsciiExceptionsTestCase(test.TestCase): def test_exception_with_non_ascii_str(self): bad_string = chr(200) - fail = misc.Failure.from_exception(ValueError(bad_string)) + fail = failure.Failure.from_exception(ValueError(bad_string)) self.assertEqual(fail.exception_str, bad_string) self.assertEqual(str(fail), 'Failure: ValueError: %s' % bad_string) def test_exception_non_ascii_unicode(self): hi_ru = u'привет' - fail = misc.Failure.from_exception(ValueError(hi_ru)) + fail = failure.Failure.from_exception(ValueError(hi_ru)) self.assertEqual(fail.exception_str, hi_ru) self.assertIsInstance(fail.exception_str, six.text_type) self.assertEqual(six.text_type(fail), @@ -268,7 +278,7 @@ class NonAsciiExceptionsTestCase(test.TestCase): hi_cn = u'嗨' fail = ValueError(hi_cn) self.assertEqual(hi_cn, exceptions.exception_message(fail)) - fail = misc.Failure.from_exception(fail) + fail = failure.Failure.from_exception(fail) wrapped_fail = exceptions.WrappedFailure([fail]) if six.PY2: # Python 2.x will unicode escape it, while python 3.3+ will not, @@ -283,12 +293,46 @@ class NonAsciiExceptionsTestCase(test.TestCase): def test_failure_equality_with_non_ascii_str(self): bad_string = chr(200) - fail = misc.Failure.from_exception(ValueError(bad_string)) + fail = failure.Failure.from_exception(ValueError(bad_string)) copied = fail.copy() self.assertEqual(fail, copied) def test_failure_equality_non_ascii_unicode(self): hi_ru = u'привет' - fail = misc.Failure.from_exception(ValueError(hi_ru)) + fail = failure.Failure.from_exception(ValueError(hi_ru)) copied = fail.copy() self.assertEqual(fail, copied) + + +class ExcInfoUtilsTest(test.TestCase): + def test_copy_none(self): + result = failure._copy_exc_info(None) + self.assertIsNone(result) + + def test_copy_exc_info(self): + exc_info = _make_exc_info("Woot!") + result = failure._copy_exc_info(exc_info) + self.assertIsNot(result, exc_info) + self.assertIs(result[0], RuntimeError) + self.assertIsNot(result[1], exc_info[1]) + self.assertIs(result[2], exc_info[2]) + + def test_none_equals(self): + self.assertTrue(failure._are_equal_exc_info_tuples(None, None)) + + def test_none_ne_tuple(self): + exc_info = _make_exc_info("Woot!") + self.assertFalse(failure._are_equal_exc_info_tuples(None, exc_info)) + + def test_tuple_nen_none(self): + exc_info = _make_exc_info("Woot!") + self.assertFalse(failure._are_equal_exc_info_tuples(exc_info, None)) + + def test_tuple_equals_itself(self): + exc_info = _make_exc_info("Woot!") + self.assertTrue(failure._are_equal_exc_info_tuples(exc_info, exc_info)) + + def test_typle_equals_copy(self): + exc_info = _make_exc_info("Woot!") + copied = failure._copy_exc_info(exc_info) + self.assertTrue(failure._are_equal_exc_info_tuples(exc_info, copied)) diff --git a/taskflow/tests/unit/test_retries.py b/taskflow/tests/unit/test_retries.py index 71ea70cb..fb1b3802 100644 --- a/taskflow/tests/unit/test_retries.py +++ b/taskflow/tests/unit/test_retries.py @@ -23,7 +23,7 @@ from taskflow import retry from taskflow import states as st from taskflow import test from taskflow.tests import utils -from taskflow.utils import misc +from taskflow.types import failure class RetryTest(utils.EngineTestBase): @@ -559,7 +559,7 @@ class RetryTest(utils.EngineTestBase): # we execute retry engine.storage.save('flow-1_retry', 1) # task fails - fail = misc.Failure.from_exception(RuntimeError('foo')), + fail = failure.Failure.from_exception(RuntimeError('foo')), engine.storage.save('task1', fail, state=st.FAILURE) if when == 'task fails': return engine @@ -635,7 +635,7 @@ class RetryTest(utils.EngineTestBase): self._make_engine(flow).run) self.assertEqual(len(r.history), 1) self.assertEqual(r.history[0][1], {}) - self.assertEqual(isinstance(r.history[0][0], misc.Failure), True) + self.assertEqual(isinstance(r.history[0][0], failure.Failure), True) def test_retry_revert_fails(self): @@ -693,7 +693,7 @@ class RetryTest(utils.EngineTestBase): engine.storage.save('test2_retry', 1) engine.storage.save('b', 11) # pretend that 'c' failed - fail = misc.Failure.from_exception(RuntimeError('Woot!')) + fail = failure.Failure.from_exception(RuntimeError('Woot!')) engine.storage.save('c', fail, st.FAILURE) engine.run() diff --git a/taskflow/tests/unit/test_utils.py b/taskflow/tests/unit/test_utils.py index 5d4b7615..c9c93f3c 100644 --- a/taskflow/tests/unit/test_utils.py +++ b/taskflow/tests/unit/test_utils.py @@ -18,7 +18,6 @@ import collections import functools import inspect import random -import sys import threading import time @@ -28,6 +27,7 @@ import testtools from taskflow import states from taskflow import test from taskflow.tests import utils as test_utils +from taskflow.types import failure from taskflow.utils import lock_utils from taskflow.utils import misc from taskflow.utils import reflection @@ -335,8 +335,8 @@ class GetClassNameTest(test.TestCase): self.assertEqual(name, 'RuntimeError') def test_global_class(self): - name = reflection.get_class_name(misc.Failure) - self.assertEqual(name, 'taskflow.utils.misc.Failure') + name = reflection.get_class_name(failure.Failure) + self.assertEqual(name, 'taskflow.types.failure.Failure') def test_class(self): name = reflection.get_class_name(Class) @@ -619,47 +619,6 @@ class UriParseTest(test.TestCase): self.assertEqual(None, parsed.password) -class ExcInfoUtilsTest(test.TestCase): - - def _make_ex_info(self): - try: - raise RuntimeError('Woot!') - except Exception: - return sys.exc_info() - - def test_copy_none(self): - result = misc.copy_exc_info(None) - self.assertIsNone(result) - - def test_copy_exc_info(self): - exc_info = self._make_ex_info() - result = misc.copy_exc_info(exc_info) - self.assertIsNot(result, exc_info) - self.assertIs(result[0], RuntimeError) - self.assertIsNot(result[1], exc_info[1]) - self.assertIs(result[2], exc_info[2]) - - def test_none_equals(self): - self.assertTrue(misc.are_equal_exc_info_tuples(None, None)) - - def test_none_ne_tuple(self): - exc_info = self._make_ex_info() - self.assertFalse(misc.are_equal_exc_info_tuples(None, exc_info)) - - def test_tuple_nen_none(self): - exc_info = self._make_ex_info() - self.assertFalse(misc.are_equal_exc_info_tuples(exc_info, None)) - - def test_tuple_equals_itself(self): - exc_info = self._make_ex_info() - self.assertTrue(misc.are_equal_exc_info_tuples(exc_info, exc_info)) - - def test_typle_equals_copy(self): - exc_info = self._make_ex_info() - copied = misc.copy_exc_info(exc_info) - self.assertTrue(misc.are_equal_exc_info_tuples(exc_info, copied)) - - class TestSequenceMinus(test.TestCase): def test_simple_case(self): diff --git a/taskflow/types/failure.py b/taskflow/types/failure.py new file mode 100644 index 00000000..93829ad6 --- /dev/null +++ b/taskflow/types/failure.py @@ -0,0 +1,287 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import sys +import traceback + +import six + +from taskflow import exceptions as exc +from taskflow.utils import reflection + + +def _copy_exc_info(exc_info): + """Make copy of exception info tuple, as deep as possible.""" + if exc_info is None: + return None + exc_type, exc_value, tb = exc_info + # NOTE(imelnikov): there is no need to copy type, and + # we can't copy traceback. + return (exc_type, copy.deepcopy(exc_value), tb) + + +def _are_equal_exc_info_tuples(ei1, ei2): + if ei1 == ei2: + return True + if ei1 is None or ei2 is None: + return False # if both are None, we returned True above + + # NOTE(imelnikov): we can't compare exceptions with '==' + # because we want exc_info be equal to it's copy made with + # copy_exc_info above. + if ei1[0] is not ei2[0]: + return False + if not all((type(ei1[1]) == type(ei2[1]), + exc.exception_message(ei1[1]) == exc.exception_message(ei2[1]), + repr(ei1[1]) == repr(ei2[1]))): + return False + if ei1[2] == ei2[2]: + return True + tb1 = traceback.format_tb(ei1[2]) + tb2 = traceback.format_tb(ei2[2]) + return tb1 == tb2 + + +class Failure(object): + """Object that represents failure. + + Failure objects encapsulate exception information so that they can be + re-used later to re-raise, inspect, examine, log, print, serialize, + deserialize... + + One example where they are dependened upon is in the WBE engine. When a + remote worker throws an exception, the WBE based engine will receive that + exception and desire to reraise it to the user/caller of the WBE based + engine for appropriate handling (this matches the behavior of non-remote + engines). To accomplish this a failure object (or a + :py:meth:`~misc.Failure.to_dict` form) would be sent over the WBE channel + and the WBE based engine would deserialize it and use this objects + :meth:`.reraise` method to cause an exception that contains + similar/equivalent information as the original exception to be reraised, + allowing the user (or the WBE engine itself) to then handle the worker + failure/exception as they desire. + + For those who are curious, here are a few reasons why the original + exception itself *may* not be reraised and instead a reraised wrapped + failure exception object will be instead. These explanations are *only* + applicable when a failure object is serialized and deserialized (when it is + retained inside the python process that the exception was created in the + the original exception can be reraised correctly without issue). + + * Traceback objects are not serializable/recreatable, since they contain + references to stack frames at the location where the exception was + raised. When a failure object is serialized and sent across a channel + and recreated it is *not* possible to restore the original traceback and + originating stack frames. + * The original exception *type* can not be guaranteed to be found, workers + can run code that is not accessible/available when the failure is being + deserialized. Even if it was possible to use pickle safely it would not + be possible to find the originating exception or associated code in this + situation. + * The original exception *type* can not be guaranteed to be constructed in + a *correct* manner. At the time of failure object creation the exception + has already been created and the failure object can not assume it has + knowledge (or the ability) to recreate the original type of the captured + exception (this is especially hard if the original exception was created + via a complex process via some custom exception constructor). + * The original exception *type* can not be guaranteed to be constructed in + a *safe* manner. Importing *foreign* exception types dynamically can be + problematic when not done correctly and in a safe manner; since failure + objects can capture any exception it would be *unsafe* to try to import + those exception types namespaces and modules on the receiver side + dynamically (this would create similar issues as the ``pickle`` module in + python has where foreign modules can be imported, causing those modules + to have code ran when this happens, and this can cause issues and + side-effects that the receiver would not have intended to have caused). + """ + DICT_VERSION = 1 + + def __init__(self, exc_info=None, **kwargs): + if not kwargs: + if exc_info is None: + exc_info = sys.exc_info() + self._exc_info = exc_info + self._exc_type_names = list( + reflection.get_all_class_names(exc_info[0], up_to=Exception)) + if not self._exc_type_names: + raise TypeError('Invalid exception type: %r' % exc_info[0]) + self._exception_str = exc.exception_message(self._exc_info[1]) + self._traceback_str = ''.join( + traceback.format_tb(self._exc_info[2])) + else: + self._exc_info = exc_info # may be None + self._exception_str = kwargs.pop('exception_str') + self._exc_type_names = kwargs.pop('exc_type_names', []) + self._traceback_str = kwargs.pop('traceback_str', None) + if kwargs: + raise TypeError( + 'Failure.__init__ got unexpected keyword argument(s): %s' + % ', '.join(six.iterkeys(kwargs))) + + @classmethod + def from_exception(cls, exception): + """Creates a failure object from a exception instance.""" + return cls((type(exception), exception, None)) + + def _matches(self, other): + if self is other: + return True + return (self._exc_type_names == other._exc_type_names + and self.exception_str == other.exception_str + and self.traceback_str == other.traceback_str) + + def matches(self, other): + """Checks if another object is equivalent to this object.""" + if not isinstance(other, Failure): + return False + if self.exc_info is None or other.exc_info is None: + return self._matches(other) + else: + return self == other + + def __eq__(self, other): + if not isinstance(other, Failure): + return NotImplemented + return (self._matches(other) and + _are_equal_exc_info_tuples(self.exc_info, other.exc_info)) + + def __ne__(self, other): + return not (self == other) + + # NOTE(imelnikov): obj.__hash__() should return same values for equal + # objects, so we should redefine __hash__. Failure equality semantics + # is a bit complicated, so for now we just mark Failure objects as + # unhashable. See python docs on object.__hash__ for more info: + # http://docs.python.org/2/reference/datamodel.html#object.__hash__ + __hash__ = None + + @property + def exception(self): + """Exception value, or None if exception value is not present. + + Exception value may be lost during serialization. + """ + if self._exc_info: + return self._exc_info[1] + else: + return None + + @property + def exception_str(self): + """String representation of exception.""" + return self._exception_str + + @property + def exc_info(self): + """Exception info tuple or None.""" + return self._exc_info + + @property + def traceback_str(self): + """Exception traceback as string.""" + return self._traceback_str + + @staticmethod + def reraise_if_any(failures): + """Re-raise exceptions if argument is not empty. + + If argument is empty list, this method returns None. If + argument is a list with a single ``Failure`` object in it, + that failure is reraised. Else, a + :class:`~taskflow.exceptions.WrappedFailure` exception + is raised with a failure list as causes. + """ + failures = list(failures) + if len(failures) == 1: + failures[0].reraise() + elif len(failures) > 1: + raise exc.WrappedFailure(failures) + + def reraise(self): + """Re-raise captured exception.""" + if self._exc_info: + six.reraise(*self._exc_info) + else: + raise exc.WrappedFailure([self]) + + def check(self, *exc_classes): + """Check if any of ``exc_classes`` caused the failure. + + Arguments of this method can be exception types or type + names (stings). If captured exception is instance of + exception of given type, the corresponding argument is + returned. Else, None is returned. + """ + for cls in exc_classes: + if isinstance(cls, type): + err = reflection.get_class_name(cls) + else: + err = cls + if err in self._exc_type_names: + return cls + return None + + def __str__(self): + return self.pformat() + + def pformat(self, traceback=False): + """Pretty formats the failure object into a string.""" + buf = six.StringIO() + buf.write( + 'Failure: %s: %s' % (self._exc_type_names[0], self._exception_str)) + if traceback: + if self._traceback_str is not None: + traceback_str = self._traceback_str.rstrip() + else: + traceback_str = None + if traceback_str: + buf.write('\nTraceback (most recent call last):\n') + buf.write(traceback_str) + else: + buf.write('\nTraceback not available.') + return buf.getvalue() + + def __iter__(self): + """Iterate over exception type names.""" + for et in self._exc_type_names: + yield et + + @classmethod + def from_dict(cls, data): + """Converts this from a dictionary to a object.""" + data = dict(data) + version = data.pop('version', None) + if version != cls.DICT_VERSION: + raise ValueError('Invalid dict version of failure object: %r' + % version) + return cls(**data) + + def to_dict(self): + """Converts this object to a dictionary.""" + return { + 'exception_str': self.exception_str, + 'traceback_str': self.traceback_str, + 'exc_type_names': list(self), + 'version': self.DICT_VERSION, + } + + def copy(self): + """Copies this object.""" + return Failure(exc_info=_copy_exc_info(self.exc_info), + exception_str=self.exception_str, + traceback_str=self.traceback_str, + exc_type_names=self._exc_type_names[:]) diff --git a/taskflow/utils/deprecation.py b/taskflow/utils/deprecation.py new file mode 100644 index 00000000..899e035d --- /dev/null +++ b/taskflow/utils/deprecation.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import warnings + +from taskflow.utils import reflection + + +def deprecation(message, stacklevel=2): + """Warns about some type of deprecation that has been made.""" + warnings.warn(message, category=DeprecationWarning, stacklevel=stacklevel) + + +# Helper accessors for the moved proxy (since it will not have easy access +# to its own getattr and setattr functions). +_setattr = object.__setattr__ +_getattr = object.__getattribute__ + + +class MovedClassProxy(object): + """Acts as a proxy to a class that was moved to another location. + + Partially based on: + + http://code.activestate.com/recipes/496741-object-proxying/ and other + various examination of how to make a good enough proxy for our usage to + move the various types we want to move during the deprecation process. + + And partially based on the wrapt object proxy (which we should just use + when it becomes available @ http://review.openstack.org/#/c/94754/). + """ + + __slots__ = [ + '__wrapped__', '__message__', '__stacklevel__', + # Ensure weakrefs can be made, + # https://docs.python.org/2/reference/datamodel.html#slots + '__weakref__', + ] + + def __init__(self, wrapped, message, stacklevel): + # We can't assign to these directly, since we are overriding getattr + # and setattr and delattr so we have to do this hoop jump to ensure + # that we don't invoke those methods (and cause infinite recursion). + _setattr(self, '__wrapped__', wrapped) + _setattr(self, '__message__', message) + _setattr(self, '__stacklevel__', stacklevel) + try: + _setattr(self, '__qualname__', wrapped.__qualname__) + except AttributeError: + pass + + def __instancecheck__(self, instance): + deprecation( + _getattr(self, '__message__'), _getattr(self, '__stacklevel__')) + return isinstance(instance, _getattr(self, '__wrapped__')) + + def __subclasscheck__(self, instance): + deprecation( + _getattr(self, '__message__'), _getattr(self, '__stacklevel__')) + return issubclass(instance, _getattr(self, '__wrapped__')) + + def __call__(self, *args, **kwargs): + deprecation( + _getattr(self, '__message__'), _getattr(self, '__stacklevel__')) + return _getattr(self, '__wrapped__')(*args, **kwargs) + + def __getattribute__(self, name): + return getattr(_getattr(self, '__wrapped__'), name) + + def __setattr__(self, name, value): + setattr(_getattr(self, '__wrapped__'), name, value) + + def __delattr__(self, name): + delattr(_getattr(self, '__wrapped__'), name) + + def __repr__(self): + wrapped = _getattr(self, '__wrapped__') + return "<%s at 0x%x for %r at 0x%x>" % ( + type(self).__name__, id(self), wrapped, id(wrapped)) + + +def moved_class(new_class, old_class_name, old_module_name, message=None, + version=None, removal_version=None): + """Deprecates a class that was moved to another location. + + This will emit warnings when the old locations class is initialized, + telling where the new and improved location for the old class now is. + """ + old_name = ".".join((old_module_name, old_class_name)) + new_name = reflection.get_class_name(new_class) + message_components = [ + "Class '%s' has moved to '%s'" % (old_name, new_name), + ] + if version: + message_components.append(" in version '%s'" % version) + if removal_version: + if removal_version == "?": + message_components.append(" and will be removed in a future" + " version") + else: + message_components.append(" and will be removed in version '%s'" + % removal_version) + if message: + message_components.append(": %s" % message) + return MovedClassProxy(new_class, "".join(message_components), 3) diff --git a/taskflow/utils/misc.py b/taskflow/utils/misc.py index 147085dd..41c5cfcd 100644 --- a/taskflow/utils/misc.py +++ b/taskflow/utils/misc.py @@ -28,7 +28,6 @@ import re import string import sys import threading -import traceback from oslo.serialization import jsonutils from oslo.utils import netutils @@ -37,7 +36,8 @@ from six.moves import map as compat_map from six.moves import range as compat_range from six.moves.urllib import parse as urlparse -from taskflow import exceptions as exc +from taskflow.types import failure +from taskflow.utils import deprecation from taskflow.utils import reflection @@ -392,6 +392,10 @@ def ensure_tree(path): raise +Failure = deprecation.moved_class(failure.Failure, 'Failure', __name__, + version="0.5", removal_version="?") + + class Notifier(object): """A notification helper class. @@ -489,38 +493,6 @@ class Notifier(object): break -def copy_exc_info(exc_info): - """Make copy of exception info tuple, as deep as possible.""" - if exc_info is None: - return None - exc_type, exc_value, tb = exc_info - # NOTE(imelnikov): there is no need to copy type, and - # we can't copy traceback. - return (exc_type, copy.deepcopy(exc_value), tb) - - -def are_equal_exc_info_tuples(ei1, ei2): - if ei1 == ei2: - return True - if ei1 is None or ei2 is None: - return False # if both are None, we returned True above - - # NOTE(imelnikov): we can't compare exceptions with '==' - # because we want exc_info be equal to it's copy made with - # copy_exc_info above. - if ei1[0] is not ei2[0]: - return False - if not all((type(ei1[1]) == type(ei2[1]), - exc.exception_message(ei1[1]) == exc.exception_message(ei2[1]), - repr(ei1[1]) == repr(ei2[1]))): - return False - if ei1[2] == ei2[2]: - return True - tb1 = traceback.format_tb(ei1[2]) - tb2 = traceback.format_tb(ei2[2]) - return tb1 == tb2 - - @contextlib.contextmanager def capture_failure(): """Captures the occurring exception and provides a failure object back. @@ -551,234 +523,3 @@ def capture_failure(): raise RuntimeError("No active exception is being handled") else: yield Failure(exc_info=exc_info) - - -class Failure(object): - """Object that represents failure. - - Failure objects encapsulate exception information so that they can be - re-used later to re-raise, inspect, examine, log, print, serialize, - deserialize... - - One example where they are dependened upon is in the WBE engine. When a - remote worker throws an exception, the WBE based engine will receive that - exception and desire to reraise it to the user/caller of the WBE based - engine for appropriate handling (this matches the behavior of non-remote - engines). To accomplish this a failure object (or a - :py:meth:`~misc.Failure.to_dict` form) would be sent over the WBE channel - and the WBE based engine would deserialize it and use this objects - :meth:`.reraise` method to cause an exception that contains - similar/equivalent information as the original exception to be reraised, - allowing the user (or the WBE engine itself) to then handle the worker - failure/exception as they desire. - - For those who are curious, here are a few reasons why the original - exception itself *may* not be reraised and instead a reraised wrapped - failure exception object will be instead. These explanations are *only* - applicable when a failure object is serialized and deserialized (when it is - retained inside the python process that the exception was created in the - the original exception can be reraised correctly without issue). - - * Traceback objects are not serializable/recreatable, since they contain - references to stack frames at the location where the exception was - raised. When a failure object is serialized and sent across a channel - and recreated it is *not* possible to restore the original traceback and - originating stack frames. - * The original exception *type* can not be guaranteed to be found, workers - can run code that is not accessible/available when the failure is being - deserialized. Even if it was possible to use pickle safely it would not - be possible to find the originating exception or associated code in this - situation. - * The original exception *type* can not be guaranteed to be constructed in - a *correct* manner. At the time of failure object creation the exception - has already been created and the failure object can not assume it has - knowledge (or the ability) to recreate the original type of the captured - exception (this is especially hard if the original exception was created - via a complex process via some custom exception constructor). - * The original exception *type* can not be guaranteed to be constructed in - a *safe* manner. Importing *foreign* exception types dynamically can be - problematic when not done correctly and in a safe manner; since failure - objects can capture any exception it would be *unsafe* to try to import - those exception types namespaces and modules on the receiver side - dynamically (this would create similar issues as the ``pickle`` module in - python has where foreign modules can be imported, causing those modules - to have code ran when this happens, and this can cause issues and - side-effects that the receiver would not have intended to have caused). - """ - DICT_VERSION = 1 - - def __init__(self, exc_info=None, **kwargs): - if not kwargs: - if exc_info is None: - exc_info = sys.exc_info() - self._exc_info = exc_info - self._exc_type_names = list( - reflection.get_all_class_names(exc_info[0], up_to=Exception)) - if not self._exc_type_names: - raise TypeError('Invalid exception type: %r' % exc_info[0]) - self._exception_str = exc.exception_message(self._exc_info[1]) - self._traceback_str = ''.join( - traceback.format_tb(self._exc_info[2])) - else: - self._exc_info = exc_info # may be None - self._exception_str = kwargs.pop('exception_str') - self._exc_type_names = kwargs.pop('exc_type_names', []) - self._traceback_str = kwargs.pop('traceback_str', None) - if kwargs: - raise TypeError( - 'Failure.__init__ got unexpected keyword argument(s): %s' - % ', '.join(six.iterkeys(kwargs))) - - @classmethod - def from_exception(cls, exception): - """Creates a failure object from a exception instance.""" - return cls((type(exception), exception, None)) - - def _matches(self, other): - if self is other: - return True - return (self._exc_type_names == other._exc_type_names - and self.exception_str == other.exception_str - and self.traceback_str == other.traceback_str) - - def matches(self, other): - """Checks if another object is equivalent to this object.""" - if not isinstance(other, Failure): - return False - if self.exc_info is None or other.exc_info is None: - return self._matches(other) - else: - return self == other - - def __eq__(self, other): - if not isinstance(other, Failure): - return NotImplemented - return (self._matches(other) and - are_equal_exc_info_tuples(self.exc_info, other.exc_info)) - - def __ne__(self, other): - return not (self == other) - - # NOTE(imelnikov): obj.__hash__() should return same values for equal - # objects, so we should redefine __hash__. Failure equality semantics - # is a bit complicated, so for now we just mark Failure objects as - # unhashable. See python docs on object.__hash__ for more info: - # http://docs.python.org/2/reference/datamodel.html#object.__hash__ - __hash__ = None - - @property - def exception(self): - """Exception value, or None if exception value is not present. - - Exception value may be lost during serialization. - """ - if self._exc_info: - return self._exc_info[1] - else: - return None - - @property - def exception_str(self): - """String representation of exception.""" - return self._exception_str - - @property - def exc_info(self): - """Exception info tuple or None.""" - return self._exc_info - - @property - def traceback_str(self): - """Exception traceback as string.""" - return self._traceback_str - - @staticmethod - def reraise_if_any(failures): - """Re-raise exceptions if argument is not empty. - - If argument is empty list, this method returns None. If - argument is a list with a single ``Failure`` object in it, - that failure is reraised. Else, a - :class:`~taskflow.exceptions.WrappedFailure` exception - is raised with a failure list as causes. - """ - failures = list(failures) - if len(failures) == 1: - failures[0].reraise() - elif len(failures) > 1: - raise exc.WrappedFailure(failures) - - def reraise(self): - """Re-raise captured exception.""" - if self._exc_info: - six.reraise(*self._exc_info) - else: - raise exc.WrappedFailure([self]) - - def check(self, *exc_classes): - """Check if any of ``exc_classes`` caused the failure. - - Arguments of this method can be exception types or type - names (stings). If captured exception is instance of - exception of given type, the corresponding argument is - returned. Else, None is returned. - """ - for cls in exc_classes: - if isinstance(cls, type): - err = reflection.get_class_name(cls) - else: - err = cls - if err in self._exc_type_names: - return cls - return None - - def __str__(self): - return self.pformat() - - def pformat(self, traceback=False): - """Pretty formats the failure object into a string.""" - buf = six.StringIO() - buf.write( - 'Failure: %s: %s' % (self._exc_type_names[0], self._exception_str)) - if traceback: - if self._traceback_str is not None: - traceback_str = self._traceback_str.rstrip() - else: - traceback_str = None - if traceback_str: - buf.write('\nTraceback (most recent call last):\n') - buf.write(traceback_str) - else: - buf.write('\nTraceback not available.') - return buf.getvalue() - - def __iter__(self): - """Iterate over exception type names.""" - for et in self._exc_type_names: - yield et - - @classmethod - def from_dict(cls, data): - """Converts this from a dictionary to a object.""" - data = dict(data) - version = data.pop('version', None) - if version != cls.DICT_VERSION: - raise ValueError('Invalid dict version of failure object: %r' - % version) - return cls(**data) - - def to_dict(self): - """Converts this object to a dictionary.""" - return { - 'exception_str': self.exception_str, - 'traceback_str': self.traceback_str, - 'exc_type_names': list(self), - 'version': self.DICT_VERSION, - } - - def copy(self): - """Copies this object.""" - return Failure(exc_info=copy_exc_info(self.exc_info), - exception_str=self.exception_str, - traceback_str=self.traceback_str, - exc_type_names=self._exc_type_names[:])