Move failure to its own type specific module
The failure module needs to be hoisted out of the misc utility file so that it can be depended on existing by users in a well defined (non-utility) location. This change does this hoisting process & creates a new module and places the existing code there, then creates a deprecated proxy that exists at the old location (this will be removed in the next version + 1). In a future change (in 0.5) we can remove this old location and remove all references to the previous location (until then we must keep the old location being used to ensure subclass checks and other types checks function properly). Part of blueprint top-level-types Change-Id: I7d13ad1e9e5f5ecc90ab81949cc92ddf7309f13c
This commit is contained in:
Joshua Harlow
committed by
Joshua Harlow
Joshua Harlow
parent
45ba27ed1a
commit
f2ea4f1288
@@ -10,7 +10,7 @@ Cache
|
||||
Failure
|
||||
=======
|
||||
|
||||
.. autoclass:: taskflow.utils.misc.Failure
|
||||
.. automodule:: taskflow.types.failure
|
||||
|
||||
FSM
|
||||
===
|
||||
|
||||
@@ -14,19 +14,29 @@
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import sys
|
||||
|
||||
import six
|
||||
|
||||
from taskflow import exceptions
|
||||
from taskflow import test
|
||||
from taskflow.tests import utils as test_utils
|
||||
from taskflow.types import failure
|
||||
from taskflow.utils import misc
|
||||
|
||||
|
||||
def _captured_failure(msg):
|
||||
try:
|
||||
raise RuntimeError(msg)
|
||||
except Exception:
|
||||
return misc.Failure()
|
||||
try:
|
||||
raise RuntimeError(msg)
|
||||
except Exception:
|
||||
return failure.Failure()
|
||||
|
||||
|
||||
def _make_exc_info(msg):
|
||||
try:
|
||||
raise RuntimeError(msg)
|
||||
except Exception:
|
||||
return sys.exc_info()
|
||||
|
||||
|
||||
class GeneralFailureObjTestsMixin(object):
|
||||
@@ -85,9 +95,9 @@ class ReCreatedFailureTestCase(test.TestCase, GeneralFailureObjTestsMixin):
|
||||
def setUp(self):
|
||||
super(ReCreatedFailureTestCase, self).setUp()
|
||||
fail_obj = _captured_failure('Woot!')
|
||||
self.fail_obj = misc.Failure(exception_str=fail_obj.exception_str,
|
||||
traceback_str=fail_obj.traceback_str,
|
||||
exc_type_names=list(fail_obj))
|
||||
self.fail_obj = failure.Failure(exception_str=fail_obj.exception_str,
|
||||
traceback_str=fail_obj.traceback_str,
|
||||
exc_type_names=list(fail_obj))
|
||||
|
||||
def test_value_lost(self):
|
||||
self.assertIs(self.fail_obj.exception, None)
|
||||
@@ -109,7 +119,7 @@ class FromExceptionTestCase(test.TestCase, GeneralFailureObjTestsMixin):
|
||||
|
||||
def setUp(self):
|
||||
super(FromExceptionTestCase, self).setUp()
|
||||
self.fail_obj = misc.Failure.from_exception(RuntimeError('Woot!'))
|
||||
self.fail_obj = failure.Failure.from_exception(RuntimeError('Woot!'))
|
||||
|
||||
def test_pformat_no_traceback(self):
|
||||
text = self.fail_obj.pformat(traceback=True)
|
||||
@@ -122,10 +132,10 @@ class FailureObjectTestCase(test.TestCase):
|
||||
try:
|
||||
raise SystemExit()
|
||||
except BaseException:
|
||||
self.assertRaises(TypeError, misc.Failure)
|
||||
self.assertRaises(TypeError, failure.Failure)
|
||||
|
||||
def test_unknown_argument(self):
|
||||
exc = self.assertRaises(TypeError, misc.Failure,
|
||||
exc = self.assertRaises(TypeError, failure.Failure,
|
||||
exception_str='Woot!',
|
||||
traceback_str=None,
|
||||
exc_type_names=['Exception'],
|
||||
@@ -134,12 +144,12 @@ class FailureObjectTestCase(test.TestCase):
|
||||
self.assertEqual(str(exc), expected)
|
||||
|
||||
def test_empty_does_not_reraise(self):
|
||||
self.assertIs(misc.Failure.reraise_if_any([]), None)
|
||||
self.assertIs(failure.Failure.reraise_if_any([]), None)
|
||||
|
||||
def test_reraises_one(self):
|
||||
fls = [_captured_failure('Woot!')]
|
||||
self.assertRaisesRegexp(RuntimeError, '^Woot!$',
|
||||
misc.Failure.reraise_if_any, fls)
|
||||
failure.Failure.reraise_if_any, fls)
|
||||
|
||||
def test_reraises_several(self):
|
||||
fls = [
|
||||
@@ -147,7 +157,7 @@ class FailureObjectTestCase(test.TestCase):
|
||||
_captured_failure('Oh, not again!')
|
||||
]
|
||||
exc = self.assertRaises(exceptions.WrappedFailure,
|
||||
misc.Failure.reraise_if_any, fls)
|
||||
failure.Failure.reraise_if_any, fls)
|
||||
self.assertEqual(list(exc), fls)
|
||||
|
||||
def test_failure_copy(self):
|
||||
@@ -160,9 +170,9 @@ class FailureObjectTestCase(test.TestCase):
|
||||
|
||||
def test_failure_copy_recaptured(self):
|
||||
captured = _captured_failure('Woot!')
|
||||
fail_obj = misc.Failure(exception_str=captured.exception_str,
|
||||
traceback_str=captured.traceback_str,
|
||||
exc_type_names=list(captured))
|
||||
fail_obj = failure.Failure(exception_str=captured.exception_str,
|
||||
traceback_str=captured.traceback_str,
|
||||
exc_type_names=list(captured))
|
||||
copied = fail_obj.copy()
|
||||
self.assertIsNot(fail_obj, copied)
|
||||
self.assertEqual(fail_obj, copied)
|
||||
@@ -171,9 +181,9 @@ class FailureObjectTestCase(test.TestCase):
|
||||
|
||||
def test_recaptured_not_eq(self):
|
||||
captured = _captured_failure('Woot!')
|
||||
fail_obj = misc.Failure(exception_str=captured.exception_str,
|
||||
traceback_str=captured.traceback_str,
|
||||
exc_type_names=list(captured))
|
||||
fail_obj = failure.Failure(exception_str=captured.exception_str,
|
||||
traceback_str=captured.traceback_str,
|
||||
exc_type_names=list(captured))
|
||||
self.assertFalse(fail_obj == captured)
|
||||
self.assertTrue(fail_obj != captured)
|
||||
self.assertTrue(fail_obj.matches(captured))
|
||||
@@ -185,13 +195,13 @@ class FailureObjectTestCase(test.TestCase):
|
||||
|
||||
def test_two_recaptured_neq(self):
|
||||
captured = _captured_failure('Woot!')
|
||||
fail_obj = misc.Failure(exception_str=captured.exception_str,
|
||||
traceback_str=captured.traceback_str,
|
||||
exc_type_names=list(captured))
|
||||
fail_obj = failure.Failure(exception_str=captured.exception_str,
|
||||
traceback_str=captured.traceback_str,
|
||||
exc_type_names=list(captured))
|
||||
new_exc_str = captured.exception_str.replace('Woot', 'w00t')
|
||||
fail_obj2 = misc.Failure(exception_str=new_exc_str,
|
||||
traceback_str=captured.traceback_str,
|
||||
exc_type_names=list(captured))
|
||||
fail_obj2 = failure.Failure(exception_str=new_exc_str,
|
||||
traceback_str=captured.traceback_str,
|
||||
exc_type_names=list(captured))
|
||||
self.assertNotEqual(fail_obj, fail_obj2)
|
||||
self.assertFalse(fail_obj2.matches(fail_obj))
|
||||
|
||||
@@ -242,7 +252,7 @@ class WrappedFailureTestCase(test.TestCase):
|
||||
try:
|
||||
raise exceptions.WrappedFailure([f1, f2])
|
||||
except Exception:
|
||||
fail_obj = misc.Failure()
|
||||
fail_obj = failure.Failure()
|
||||
|
||||
wf = exceptions.WrappedFailure([fail_obj, f3])
|
||||
self.assertEqual(list(wf), [f1, f2, f3])
|
||||
@@ -252,13 +262,13 @@ class NonAsciiExceptionsTestCase(test.TestCase):
|
||||
|
||||
def test_exception_with_non_ascii_str(self):
|
||||
bad_string = chr(200)
|
||||
fail = misc.Failure.from_exception(ValueError(bad_string))
|
||||
fail = failure.Failure.from_exception(ValueError(bad_string))
|
||||
self.assertEqual(fail.exception_str, bad_string)
|
||||
self.assertEqual(str(fail), 'Failure: ValueError: %s' % bad_string)
|
||||
|
||||
def test_exception_non_ascii_unicode(self):
|
||||
hi_ru = u'привет'
|
||||
fail = misc.Failure.from_exception(ValueError(hi_ru))
|
||||
fail = failure.Failure.from_exception(ValueError(hi_ru))
|
||||
self.assertEqual(fail.exception_str, hi_ru)
|
||||
self.assertIsInstance(fail.exception_str, six.text_type)
|
||||
self.assertEqual(six.text_type(fail),
|
||||
@@ -268,7 +278,7 @@ class NonAsciiExceptionsTestCase(test.TestCase):
|
||||
hi_cn = u'嗨'
|
||||
fail = ValueError(hi_cn)
|
||||
self.assertEqual(hi_cn, exceptions.exception_message(fail))
|
||||
fail = misc.Failure.from_exception(fail)
|
||||
fail = failure.Failure.from_exception(fail)
|
||||
wrapped_fail = exceptions.WrappedFailure([fail])
|
||||
if six.PY2:
|
||||
# Python 2.x will unicode escape it, while python 3.3+ will not,
|
||||
@@ -283,12 +293,46 @@ class NonAsciiExceptionsTestCase(test.TestCase):
|
||||
|
||||
def test_failure_equality_with_non_ascii_str(self):
|
||||
bad_string = chr(200)
|
||||
fail = misc.Failure.from_exception(ValueError(bad_string))
|
||||
fail = failure.Failure.from_exception(ValueError(bad_string))
|
||||
copied = fail.copy()
|
||||
self.assertEqual(fail, copied)
|
||||
|
||||
def test_failure_equality_non_ascii_unicode(self):
|
||||
hi_ru = u'привет'
|
||||
fail = misc.Failure.from_exception(ValueError(hi_ru))
|
||||
fail = failure.Failure.from_exception(ValueError(hi_ru))
|
||||
copied = fail.copy()
|
||||
self.assertEqual(fail, copied)
|
||||
|
||||
|
||||
class ExcInfoUtilsTest(test.TestCase):
|
||||
def test_copy_none(self):
|
||||
result = failure._copy_exc_info(None)
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_copy_exc_info(self):
|
||||
exc_info = _make_exc_info("Woot!")
|
||||
result = failure._copy_exc_info(exc_info)
|
||||
self.assertIsNot(result, exc_info)
|
||||
self.assertIs(result[0], RuntimeError)
|
||||
self.assertIsNot(result[1], exc_info[1])
|
||||
self.assertIs(result[2], exc_info[2])
|
||||
|
||||
def test_none_equals(self):
|
||||
self.assertTrue(failure._are_equal_exc_info_tuples(None, None))
|
||||
|
||||
def test_none_ne_tuple(self):
|
||||
exc_info = _make_exc_info("Woot!")
|
||||
self.assertFalse(failure._are_equal_exc_info_tuples(None, exc_info))
|
||||
|
||||
def test_tuple_nen_none(self):
|
||||
exc_info = _make_exc_info("Woot!")
|
||||
self.assertFalse(failure._are_equal_exc_info_tuples(exc_info, None))
|
||||
|
||||
def test_tuple_equals_itself(self):
|
||||
exc_info = _make_exc_info("Woot!")
|
||||
self.assertTrue(failure._are_equal_exc_info_tuples(exc_info, exc_info))
|
||||
|
||||
def test_typle_equals_copy(self):
|
||||
exc_info = _make_exc_info("Woot!")
|
||||
copied = failure._copy_exc_info(exc_info)
|
||||
self.assertTrue(failure._are_equal_exc_info_tuples(exc_info, copied))
|
||||
@@ -23,7 +23,7 @@ from taskflow import retry
|
||||
from taskflow import states as st
|
||||
from taskflow import test
|
||||
from taskflow.tests import utils
|
||||
from taskflow.utils import misc
|
||||
from taskflow.types import failure
|
||||
|
||||
|
||||
class RetryTest(utils.EngineTestBase):
|
||||
@@ -559,7 +559,7 @@ class RetryTest(utils.EngineTestBase):
|
||||
# we execute retry
|
||||
engine.storage.save('flow-1_retry', 1)
|
||||
# task fails
|
||||
fail = misc.Failure.from_exception(RuntimeError('foo')),
|
||||
fail = failure.Failure.from_exception(RuntimeError('foo')),
|
||||
engine.storage.save('task1', fail, state=st.FAILURE)
|
||||
if when == 'task fails':
|
||||
return engine
|
||||
@@ -635,7 +635,7 @@ class RetryTest(utils.EngineTestBase):
|
||||
self._make_engine(flow).run)
|
||||
self.assertEqual(len(r.history), 1)
|
||||
self.assertEqual(r.history[0][1], {})
|
||||
self.assertEqual(isinstance(r.history[0][0], misc.Failure), True)
|
||||
self.assertEqual(isinstance(r.history[0][0], failure.Failure), True)
|
||||
|
||||
def test_retry_revert_fails(self):
|
||||
|
||||
@@ -693,7 +693,7 @@ class RetryTest(utils.EngineTestBase):
|
||||
engine.storage.save('test2_retry', 1)
|
||||
engine.storage.save('b', 11)
|
||||
# pretend that 'c' failed
|
||||
fail = misc.Failure.from_exception(RuntimeError('Woot!'))
|
||||
fail = failure.Failure.from_exception(RuntimeError('Woot!'))
|
||||
engine.storage.save('c', fail, st.FAILURE)
|
||||
|
||||
engine.run()
|
||||
|
||||
@@ -18,7 +18,6 @@ import collections
|
||||
import functools
|
||||
import inspect
|
||||
import random
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
@@ -28,6 +27,7 @@ import testtools
|
||||
from taskflow import states
|
||||
from taskflow import test
|
||||
from taskflow.tests import utils as test_utils
|
||||
from taskflow.types import failure
|
||||
from taskflow.utils import lock_utils
|
||||
from taskflow.utils import misc
|
||||
from taskflow.utils import reflection
|
||||
@@ -335,8 +335,8 @@ class GetClassNameTest(test.TestCase):
|
||||
self.assertEqual(name, 'RuntimeError')
|
||||
|
||||
def test_global_class(self):
|
||||
name = reflection.get_class_name(misc.Failure)
|
||||
self.assertEqual(name, 'taskflow.utils.misc.Failure')
|
||||
name = reflection.get_class_name(failure.Failure)
|
||||
self.assertEqual(name, 'taskflow.types.failure.Failure')
|
||||
|
||||
def test_class(self):
|
||||
name = reflection.get_class_name(Class)
|
||||
@@ -619,47 +619,6 @@ class UriParseTest(test.TestCase):
|
||||
self.assertEqual(None, parsed.password)
|
||||
|
||||
|
||||
class ExcInfoUtilsTest(test.TestCase):
|
||||
|
||||
def _make_ex_info(self):
|
||||
try:
|
||||
raise RuntimeError('Woot!')
|
||||
except Exception:
|
||||
return sys.exc_info()
|
||||
|
||||
def test_copy_none(self):
|
||||
result = misc.copy_exc_info(None)
|
||||
self.assertIsNone(result)
|
||||
|
||||
def test_copy_exc_info(self):
|
||||
exc_info = self._make_ex_info()
|
||||
result = misc.copy_exc_info(exc_info)
|
||||
self.assertIsNot(result, exc_info)
|
||||
self.assertIs(result[0], RuntimeError)
|
||||
self.assertIsNot(result[1], exc_info[1])
|
||||
self.assertIs(result[2], exc_info[2])
|
||||
|
||||
def test_none_equals(self):
|
||||
self.assertTrue(misc.are_equal_exc_info_tuples(None, None))
|
||||
|
||||
def test_none_ne_tuple(self):
|
||||
exc_info = self._make_ex_info()
|
||||
self.assertFalse(misc.are_equal_exc_info_tuples(None, exc_info))
|
||||
|
||||
def test_tuple_nen_none(self):
|
||||
exc_info = self._make_ex_info()
|
||||
self.assertFalse(misc.are_equal_exc_info_tuples(exc_info, None))
|
||||
|
||||
def test_tuple_equals_itself(self):
|
||||
exc_info = self._make_ex_info()
|
||||
self.assertTrue(misc.are_equal_exc_info_tuples(exc_info, exc_info))
|
||||
|
||||
def test_typle_equals_copy(self):
|
||||
exc_info = self._make_ex_info()
|
||||
copied = misc.copy_exc_info(exc_info)
|
||||
self.assertTrue(misc.are_equal_exc_info_tuples(exc_info, copied))
|
||||
|
||||
|
||||
class TestSequenceMinus(test.TestCase):
|
||||
|
||||
def test_simple_case(self):
|
||||
|
||||
287
taskflow/types/failure.py
Normal file
287
taskflow/types/failure.py
Normal file
@@ -0,0 +1,287 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import copy
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
import six
|
||||
|
||||
from taskflow import exceptions as exc
|
||||
from taskflow.utils import reflection
|
||||
|
||||
|
||||
def _copy_exc_info(exc_info):
|
||||
"""Make copy of exception info tuple, as deep as possible."""
|
||||
if exc_info is None:
|
||||
return None
|
||||
exc_type, exc_value, tb = exc_info
|
||||
# NOTE(imelnikov): there is no need to copy type, and
|
||||
# we can't copy traceback.
|
||||
return (exc_type, copy.deepcopy(exc_value), tb)
|
||||
|
||||
|
||||
def _are_equal_exc_info_tuples(ei1, ei2):
|
||||
if ei1 == ei2:
|
||||
return True
|
||||
if ei1 is None or ei2 is None:
|
||||
return False # if both are None, we returned True above
|
||||
|
||||
# NOTE(imelnikov): we can't compare exceptions with '=='
|
||||
# because we want exc_info be equal to it's copy made with
|
||||
# copy_exc_info above.
|
||||
if ei1[0] is not ei2[0]:
|
||||
return False
|
||||
if not all((type(ei1[1]) == type(ei2[1]),
|
||||
exc.exception_message(ei1[1]) == exc.exception_message(ei2[1]),
|
||||
repr(ei1[1]) == repr(ei2[1]))):
|
||||
return False
|
||||
if ei1[2] == ei2[2]:
|
||||
return True
|
||||
tb1 = traceback.format_tb(ei1[2])
|
||||
tb2 = traceback.format_tb(ei2[2])
|
||||
return tb1 == tb2
|
||||
|
||||
|
||||
class Failure(object):
|
||||
"""Object that represents failure.
|
||||
|
||||
Failure objects encapsulate exception information so that they can be
|
||||
re-used later to re-raise, inspect, examine, log, print, serialize,
|
||||
deserialize...
|
||||
|
||||
One example where they are dependened upon is in the WBE engine. When a
|
||||
remote worker throws an exception, the WBE based engine will receive that
|
||||
exception and desire to reraise it to the user/caller of the WBE based
|
||||
engine for appropriate handling (this matches the behavior of non-remote
|
||||
engines). To accomplish this a failure object (or a
|
||||
:py:meth:`~misc.Failure.to_dict` form) would be sent over the WBE channel
|
||||
and the WBE based engine would deserialize it and use this objects
|
||||
:meth:`.reraise` method to cause an exception that contains
|
||||
similar/equivalent information as the original exception to be reraised,
|
||||
allowing the user (or the WBE engine itself) to then handle the worker
|
||||
failure/exception as they desire.
|
||||
|
||||
For those who are curious, here are a few reasons why the original
|
||||
exception itself *may* not be reraised and instead a reraised wrapped
|
||||
failure exception object will be instead. These explanations are *only*
|
||||
applicable when a failure object is serialized and deserialized (when it is
|
||||
retained inside the python process that the exception was created in the
|
||||
the original exception can be reraised correctly without issue).
|
||||
|
||||
* Traceback objects are not serializable/recreatable, since they contain
|
||||
references to stack frames at the location where the exception was
|
||||
raised. When a failure object is serialized and sent across a channel
|
||||
and recreated it is *not* possible to restore the original traceback and
|
||||
originating stack frames.
|
||||
* The original exception *type* can not be guaranteed to be found, workers
|
||||
can run code that is not accessible/available when the failure is being
|
||||
deserialized. Even if it was possible to use pickle safely it would not
|
||||
be possible to find the originating exception or associated code in this
|
||||
situation.
|
||||
* The original exception *type* can not be guaranteed to be constructed in
|
||||
a *correct* manner. At the time of failure object creation the exception
|
||||
has already been created and the failure object can not assume it has
|
||||
knowledge (or the ability) to recreate the original type of the captured
|
||||
exception (this is especially hard if the original exception was created
|
||||
via a complex process via some custom exception constructor).
|
||||
* The original exception *type* can not be guaranteed to be constructed in
|
||||
a *safe* manner. Importing *foreign* exception types dynamically can be
|
||||
problematic when not done correctly and in a safe manner; since failure
|
||||
objects can capture any exception it would be *unsafe* to try to import
|
||||
those exception types namespaces and modules on the receiver side
|
||||
dynamically (this would create similar issues as the ``pickle`` module in
|
||||
python has where foreign modules can be imported, causing those modules
|
||||
to have code ran when this happens, and this can cause issues and
|
||||
side-effects that the receiver would not have intended to have caused).
|
||||
"""
|
||||
DICT_VERSION = 1
|
||||
|
||||
def __init__(self, exc_info=None, **kwargs):
|
||||
if not kwargs:
|
||||
if exc_info is None:
|
||||
exc_info = sys.exc_info()
|
||||
self._exc_info = exc_info
|
||||
self._exc_type_names = list(
|
||||
reflection.get_all_class_names(exc_info[0], up_to=Exception))
|
||||
if not self._exc_type_names:
|
||||
raise TypeError('Invalid exception type: %r' % exc_info[0])
|
||||
self._exception_str = exc.exception_message(self._exc_info[1])
|
||||
self._traceback_str = ''.join(
|
||||
traceback.format_tb(self._exc_info[2]))
|
||||
else:
|
||||
self._exc_info = exc_info # may be None
|
||||
self._exception_str = kwargs.pop('exception_str')
|
||||
self._exc_type_names = kwargs.pop('exc_type_names', [])
|
||||
self._traceback_str = kwargs.pop('traceback_str', None)
|
||||
if kwargs:
|
||||
raise TypeError(
|
||||
'Failure.__init__ got unexpected keyword argument(s): %s'
|
||||
% ', '.join(six.iterkeys(kwargs)))
|
||||
|
||||
@classmethod
|
||||
def from_exception(cls, exception):
|
||||
"""Creates a failure object from a exception instance."""
|
||||
return cls((type(exception), exception, None))
|
||||
|
||||
def _matches(self, other):
|
||||
if self is other:
|
||||
return True
|
||||
return (self._exc_type_names == other._exc_type_names
|
||||
and self.exception_str == other.exception_str
|
||||
and self.traceback_str == other.traceback_str)
|
||||
|
||||
def matches(self, other):
|
||||
"""Checks if another object is equivalent to this object."""
|
||||
if not isinstance(other, Failure):
|
||||
return False
|
||||
if self.exc_info is None or other.exc_info is None:
|
||||
return self._matches(other)
|
||||
else:
|
||||
return self == other
|
||||
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, Failure):
|
||||
return NotImplemented
|
||||
return (self._matches(other) and
|
||||
_are_equal_exc_info_tuples(self.exc_info, other.exc_info))
|
||||
|
||||
def __ne__(self, other):
|
||||
return not (self == other)
|
||||
|
||||
# NOTE(imelnikov): obj.__hash__() should return same values for equal
|
||||
# objects, so we should redefine __hash__. Failure equality semantics
|
||||
# is a bit complicated, so for now we just mark Failure objects as
|
||||
# unhashable. See python docs on object.__hash__ for more info:
|
||||
# http://docs.python.org/2/reference/datamodel.html#object.__hash__
|
||||
__hash__ = None
|
||||
|
||||
@property
|
||||
def exception(self):
|
||||
"""Exception value, or None if exception value is not present.
|
||||
|
||||
Exception value may be lost during serialization.
|
||||
"""
|
||||
if self._exc_info:
|
||||
return self._exc_info[1]
|
||||
else:
|
||||
return None
|
||||
|
||||
@property
|
||||
def exception_str(self):
|
||||
"""String representation of exception."""
|
||||
return self._exception_str
|
||||
|
||||
@property
|
||||
def exc_info(self):
|
||||
"""Exception info tuple or None."""
|
||||
return self._exc_info
|
||||
|
||||
@property
|
||||
def traceback_str(self):
|
||||
"""Exception traceback as string."""
|
||||
return self._traceback_str
|
||||
|
||||
@staticmethod
|
||||
def reraise_if_any(failures):
|
||||
"""Re-raise exceptions if argument is not empty.
|
||||
|
||||
If argument is empty list, this method returns None. If
|
||||
argument is a list with a single ``Failure`` object in it,
|
||||
that failure is reraised. Else, a
|
||||
:class:`~taskflow.exceptions.WrappedFailure` exception
|
||||
is raised with a failure list as causes.
|
||||
"""
|
||||
failures = list(failures)
|
||||
if len(failures) == 1:
|
||||
failures[0].reraise()
|
||||
elif len(failures) > 1:
|
||||
raise exc.WrappedFailure(failures)
|
||||
|
||||
def reraise(self):
|
||||
"""Re-raise captured exception."""
|
||||
if self._exc_info:
|
||||
six.reraise(*self._exc_info)
|
||||
else:
|
||||
raise exc.WrappedFailure([self])
|
||||
|
||||
def check(self, *exc_classes):
|
||||
"""Check if any of ``exc_classes`` caused the failure.
|
||||
|
||||
Arguments of this method can be exception types or type
|
||||
names (stings). If captured exception is instance of
|
||||
exception of given type, the corresponding argument is
|
||||
returned. Else, None is returned.
|
||||
"""
|
||||
for cls in exc_classes:
|
||||
if isinstance(cls, type):
|
||||
err = reflection.get_class_name(cls)
|
||||
else:
|
||||
err = cls
|
||||
if err in self._exc_type_names:
|
||||
return cls
|
||||
return None
|
||||
|
||||
def __str__(self):
|
||||
return self.pformat()
|
||||
|
||||
def pformat(self, traceback=False):
|
||||
"""Pretty formats the failure object into a string."""
|
||||
buf = six.StringIO()
|
||||
buf.write(
|
||||
'Failure: %s: %s' % (self._exc_type_names[0], self._exception_str))
|
||||
if traceback:
|
||||
if self._traceback_str is not None:
|
||||
traceback_str = self._traceback_str.rstrip()
|
||||
else:
|
||||
traceback_str = None
|
||||
if traceback_str:
|
||||
buf.write('\nTraceback (most recent call last):\n')
|
||||
buf.write(traceback_str)
|
||||
else:
|
||||
buf.write('\nTraceback not available.')
|
||||
return buf.getvalue()
|
||||
|
||||
def __iter__(self):
|
||||
"""Iterate over exception type names."""
|
||||
for et in self._exc_type_names:
|
||||
yield et
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data):
|
||||
"""Converts this from a dictionary to a object."""
|
||||
data = dict(data)
|
||||
version = data.pop('version', None)
|
||||
if version != cls.DICT_VERSION:
|
||||
raise ValueError('Invalid dict version of failure object: %r'
|
||||
% version)
|
||||
return cls(**data)
|
||||
|
||||
def to_dict(self):
|
||||
"""Converts this object to a dictionary."""
|
||||
return {
|
||||
'exception_str': self.exception_str,
|
||||
'traceback_str': self.traceback_str,
|
||||
'exc_type_names': list(self),
|
||||
'version': self.DICT_VERSION,
|
||||
}
|
||||
|
||||
def copy(self):
|
||||
"""Copies this object."""
|
||||
return Failure(exc_info=_copy_exc_info(self.exc_info),
|
||||
exception_str=self.exception_str,
|
||||
traceback_str=self.traceback_str,
|
||||
exc_type_names=self._exc_type_names[:])
|
||||
118
taskflow/utils/deprecation.py
Normal file
118
taskflow/utils/deprecation.py
Normal file
@@ -0,0 +1,118 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# Copyright (C) 2014 Yahoo! Inc. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import warnings
|
||||
|
||||
from taskflow.utils import reflection
|
||||
|
||||
|
||||
def deprecation(message, stacklevel=2):
|
||||
"""Warns about some type of deprecation that has been made."""
|
||||
warnings.warn(message, category=DeprecationWarning, stacklevel=stacklevel)
|
||||
|
||||
|
||||
# Helper accessors for the moved proxy (since it will not have easy access
|
||||
# to its own getattr and setattr functions).
|
||||
_setattr = object.__setattr__
|
||||
_getattr = object.__getattribute__
|
||||
|
||||
|
||||
class MovedClassProxy(object):
|
||||
"""Acts as a proxy to a class that was moved to another location.
|
||||
|
||||
Partially based on:
|
||||
|
||||
http://code.activestate.com/recipes/496741-object-proxying/ and other
|
||||
various examination of how to make a good enough proxy for our usage to
|
||||
move the various types we want to move during the deprecation process.
|
||||
|
||||
And partially based on the wrapt object proxy (which we should just use
|
||||
when it becomes available @ http://review.openstack.org/#/c/94754/).
|
||||
"""
|
||||
|
||||
__slots__ = [
|
||||
'__wrapped__', '__message__', '__stacklevel__',
|
||||
# Ensure weakrefs can be made,
|
||||
# https://docs.python.org/2/reference/datamodel.html#slots
|
||||
'__weakref__',
|
||||
]
|
||||
|
||||
def __init__(self, wrapped, message, stacklevel):
|
||||
# We can't assign to these directly, since we are overriding getattr
|
||||
# and setattr and delattr so we have to do this hoop jump to ensure
|
||||
# that we don't invoke those methods (and cause infinite recursion).
|
||||
_setattr(self, '__wrapped__', wrapped)
|
||||
_setattr(self, '__message__', message)
|
||||
_setattr(self, '__stacklevel__', stacklevel)
|
||||
try:
|
||||
_setattr(self, '__qualname__', wrapped.__qualname__)
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
def __instancecheck__(self, instance):
|
||||
deprecation(
|
||||
_getattr(self, '__message__'), _getattr(self, '__stacklevel__'))
|
||||
return isinstance(instance, _getattr(self, '__wrapped__'))
|
||||
|
||||
def __subclasscheck__(self, instance):
|
||||
deprecation(
|
||||
_getattr(self, '__message__'), _getattr(self, '__stacklevel__'))
|
||||
return issubclass(instance, _getattr(self, '__wrapped__'))
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
deprecation(
|
||||
_getattr(self, '__message__'), _getattr(self, '__stacklevel__'))
|
||||
return _getattr(self, '__wrapped__')(*args, **kwargs)
|
||||
|
||||
def __getattribute__(self, name):
|
||||
return getattr(_getattr(self, '__wrapped__'), name)
|
||||
|
||||
def __setattr__(self, name, value):
|
||||
setattr(_getattr(self, '__wrapped__'), name, value)
|
||||
|
||||
def __delattr__(self, name):
|
||||
delattr(_getattr(self, '__wrapped__'), name)
|
||||
|
||||
def __repr__(self):
|
||||
wrapped = _getattr(self, '__wrapped__')
|
||||
return "<%s at 0x%x for %r at 0x%x>" % (
|
||||
type(self).__name__, id(self), wrapped, id(wrapped))
|
||||
|
||||
|
||||
def moved_class(new_class, old_class_name, old_module_name, message=None,
|
||||
version=None, removal_version=None):
|
||||
"""Deprecates a class that was moved to another location.
|
||||
|
||||
This will emit warnings when the old locations class is initialized,
|
||||
telling where the new and improved location for the old class now is.
|
||||
"""
|
||||
old_name = ".".join((old_module_name, old_class_name))
|
||||
new_name = reflection.get_class_name(new_class)
|
||||
message_components = [
|
||||
"Class '%s' has moved to '%s'" % (old_name, new_name),
|
||||
]
|
||||
if version:
|
||||
message_components.append(" in version '%s'" % version)
|
||||
if removal_version:
|
||||
if removal_version == "?":
|
||||
message_components.append(" and will be removed in a future"
|
||||
" version")
|
||||
else:
|
||||
message_components.append(" and will be removed in version '%s'"
|
||||
% removal_version)
|
||||
if message:
|
||||
message_components.append(": %s" % message)
|
||||
return MovedClassProxy(new_class, "".join(message_components), 3)
|
||||
@@ -28,7 +28,6 @@ import re
|
||||
import string
|
||||
import sys
|
||||
import threading
|
||||
import traceback
|
||||
|
||||
from oslo.serialization import jsonutils
|
||||
from oslo.utils import netutils
|
||||
@@ -37,7 +36,8 @@ from six.moves import map as compat_map
|
||||
from six.moves import range as compat_range
|
||||
from six.moves.urllib import parse as urlparse
|
||||
|
||||
from taskflow import exceptions as exc
|
||||
from taskflow.types import failure
|
||||
from taskflow.utils import deprecation
|
||||
from taskflow.utils import reflection
|
||||
|
||||
|
||||
@@ -392,6 +392,10 @@ def ensure_tree(path):
|
||||
raise
|
||||
|
||||
|
||||
Failure = deprecation.moved_class(failure.Failure, 'Failure', __name__,
|
||||
version="0.5", removal_version="?")
|
||||
|
||||
|
||||
class Notifier(object):
|
||||
"""A notification helper class.
|
||||
|
||||
@@ -489,38 +493,6 @@ class Notifier(object):
|
||||
break
|
||||
|
||||
|
||||
def copy_exc_info(exc_info):
|
||||
"""Make copy of exception info tuple, as deep as possible."""
|
||||
if exc_info is None:
|
||||
return None
|
||||
exc_type, exc_value, tb = exc_info
|
||||
# NOTE(imelnikov): there is no need to copy type, and
|
||||
# we can't copy traceback.
|
||||
return (exc_type, copy.deepcopy(exc_value), tb)
|
||||
|
||||
|
||||
def are_equal_exc_info_tuples(ei1, ei2):
|
||||
if ei1 == ei2:
|
||||
return True
|
||||
if ei1 is None or ei2 is None:
|
||||
return False # if both are None, we returned True above
|
||||
|
||||
# NOTE(imelnikov): we can't compare exceptions with '=='
|
||||
# because we want exc_info be equal to it's copy made with
|
||||
# copy_exc_info above.
|
||||
if ei1[0] is not ei2[0]:
|
||||
return False
|
||||
if not all((type(ei1[1]) == type(ei2[1]),
|
||||
exc.exception_message(ei1[1]) == exc.exception_message(ei2[1]),
|
||||
repr(ei1[1]) == repr(ei2[1]))):
|
||||
return False
|
||||
if ei1[2] == ei2[2]:
|
||||
return True
|
||||
tb1 = traceback.format_tb(ei1[2])
|
||||
tb2 = traceback.format_tb(ei2[2])
|
||||
return tb1 == tb2
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def capture_failure():
|
||||
"""Captures the occurring exception and provides a failure object back.
|
||||
@@ -551,234 +523,3 @@ def capture_failure():
|
||||
raise RuntimeError("No active exception is being handled")
|
||||
else:
|
||||
yield Failure(exc_info=exc_info)
|
||||
|
||||
|
||||
class Failure(object):
|
||||
"""Object that represents failure.
|
||||
|
||||
Failure objects encapsulate exception information so that they can be
|
||||
re-used later to re-raise, inspect, examine, log, print, serialize,
|
||||
deserialize...
|
||||
|
||||
One example where they are dependened upon is in the WBE engine. When a
|
||||
remote worker throws an exception, the WBE based engine will receive that
|
||||
exception and desire to reraise it to the user/caller of the WBE based
|
||||
engine for appropriate handling (this matches the behavior of non-remote
|
||||
engines). To accomplish this a failure object (or a
|
||||
:py:meth:`~misc.Failure.to_dict` form) would be sent over the WBE channel
|
||||
and the WBE based engine would deserialize it and use this objects
|
||||
:meth:`.reraise` method to cause an exception that contains
|
||||
similar/equivalent information as the original exception to be reraised,
|
||||
allowing the user (or the WBE engine itself) to then handle the worker
|
||||
failure/exception as they desire.
|
||||
|
||||
For those who are curious, here are a few reasons why the original
|
||||
exception itself *may* not be reraised and instead a reraised wrapped
|
||||
failure exception object will be instead. These explanations are *only*
|
||||
applicable when a failure object is serialized and deserialized (when it is
|
||||
retained inside the python process that the exception was created in the
|
||||
the original exception can be reraised correctly without issue).
|
||||
|
||||
* Traceback objects are not serializable/recreatable, since they contain
|
||||
references to stack frames at the location where the exception was
|
||||
raised. When a failure object is serialized and sent across a channel
|
||||
and recreated it is *not* possible to restore the original traceback and
|
||||
originating stack frames.
|
||||
* The original exception *type* can not be guaranteed to be found, workers
|
||||
can run code that is not accessible/available when the failure is being
|
||||
deserialized. Even if it was possible to use pickle safely it would not
|
||||
be possible to find the originating exception or associated code in this
|
||||
situation.
|
||||
* The original exception *type* can not be guaranteed to be constructed in
|
||||
a *correct* manner. At the time of failure object creation the exception
|
||||
has already been created and the failure object can not assume it has
|
||||
knowledge (or the ability) to recreate the original type of the captured
|
||||
exception (this is especially hard if the original exception was created
|
||||
via a complex process via some custom exception constructor).
|
||||
* The original exception *type* can not be guaranteed to be constructed in
|
||||
a *safe* manner. Importing *foreign* exception types dynamically can be
|
||||
problematic when not done correctly and in a safe manner; since failure
|
||||
objects can capture any exception it would be *unsafe* to try to import
|
||||
those exception types namespaces and modules on the receiver side
|
||||
dynamically (this would create similar issues as the ``pickle`` module in
|
||||
python has where foreign modules can be imported, causing those modules
|
||||
to have code ran when this happens, and this can cause issues and
|
||||
side-effects that the receiver would not have intended to have caused).
|
||||
"""
|
||||
DICT_VERSION = 1
|
||||
|
||||
def __init__(self, exc_info=None, **kwargs):
|
||||
if not kwargs:
|
||||
if exc_info is None:
|
||||
exc_info = sys.exc_info()
|
||||
self._exc_info = exc_info
|
||||
self._exc_type_names = list(
|
||||
reflection.get_all_class_names(exc_info[0], up_to=Exception))
|
||||
if not self._exc_type_names:
|
||||
raise TypeError('Invalid exception type: %r' % exc_info[0])
|
||||
self._exception_str = exc.exception_message(self._exc_info[1])
|
||||
self._traceback_str = ''.join(
|
||||
traceback.format_tb(self._exc_info[2]))
|
||||
else:
|
||||
self._exc_info = exc_info # may be None
|
||||
self._exception_str = kwargs.pop('exception_str')
|
||||
self._exc_type_names = kwargs.pop('exc_type_names', [])
|
||||
self._traceback_str = kwargs.pop('traceback_str', None)
|
||||
if kwargs:
|
||||
raise TypeError(
|
||||
'Failure.__init__ got unexpected keyword argument(s): %s'
|
||||
% ', '.join(six.iterkeys(kwargs)))
|
||||
|
||||
@classmethod
|
||||
def from_exception(cls, exception):
|
||||
"""Creates a failure object from a exception instance."""
|
||||
return cls((type(exception), exception, None))
|
||||
|
||||
def _matches(self, other):
|
||||
if self is other:
|
||||
return True
|
||||
return (self._exc_type_names == other._exc_type_names
|
||||
and self.exception_str == other.exception_str
|
||||
and self.traceback_str == other.traceback_str)
|
||||
|
||||
def matches(self, other):
|
||||
"""Checks if another object is equivalent to this object."""
|
||||
if not isinstance(other, Failure):
|
||||
return False
|
||||
if self.exc_info is None or other.exc_info is None:
|
||||
return self._matches(other)
|
||||
else:
|
||||
return self == other
|
||||
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, Failure):
|
||||
return NotImplemented
|
||||
return (self._matches(other) and
|
||||
are_equal_exc_info_tuples(self.exc_info, other.exc_info))
|
||||
|
||||
def __ne__(self, other):
|
||||
return not (self == other)
|
||||
|
||||
# NOTE(imelnikov): obj.__hash__() should return same values for equal
|
||||
# objects, so we should redefine __hash__. Failure equality semantics
|
||||
# is a bit complicated, so for now we just mark Failure objects as
|
||||
# unhashable. See python docs on object.__hash__ for more info:
|
||||
# http://docs.python.org/2/reference/datamodel.html#object.__hash__
|
||||
__hash__ = None
|
||||
|
||||
@property
|
||||
def exception(self):
|
||||
"""Exception value, or None if exception value is not present.
|
||||
|
||||
Exception value may be lost during serialization.
|
||||
"""
|
||||
if self._exc_info:
|
||||
return self._exc_info[1]
|
||||
else:
|
||||
return None
|
||||
|
||||
@property
|
||||
def exception_str(self):
|
||||
"""String representation of exception."""
|
||||
return self._exception_str
|
||||
|
||||
@property
|
||||
def exc_info(self):
|
||||
"""Exception info tuple or None."""
|
||||
return self._exc_info
|
||||
|
||||
@property
|
||||
def traceback_str(self):
|
||||
"""Exception traceback as string."""
|
||||
return self._traceback_str
|
||||
|
||||
@staticmethod
|
||||
def reraise_if_any(failures):
|
||||
"""Re-raise exceptions if argument is not empty.
|
||||
|
||||
If argument is empty list, this method returns None. If
|
||||
argument is a list with a single ``Failure`` object in it,
|
||||
that failure is reraised. Else, a
|
||||
:class:`~taskflow.exceptions.WrappedFailure` exception
|
||||
is raised with a failure list as causes.
|
||||
"""
|
||||
failures = list(failures)
|
||||
if len(failures) == 1:
|
||||
failures[0].reraise()
|
||||
elif len(failures) > 1:
|
||||
raise exc.WrappedFailure(failures)
|
||||
|
||||
def reraise(self):
|
||||
"""Re-raise captured exception."""
|
||||
if self._exc_info:
|
||||
six.reraise(*self._exc_info)
|
||||
else:
|
||||
raise exc.WrappedFailure([self])
|
||||
|
||||
def check(self, *exc_classes):
|
||||
"""Check if any of ``exc_classes`` caused the failure.
|
||||
|
||||
Arguments of this method can be exception types or type
|
||||
names (stings). If captured exception is instance of
|
||||
exception of given type, the corresponding argument is
|
||||
returned. Else, None is returned.
|
||||
"""
|
||||
for cls in exc_classes:
|
||||
if isinstance(cls, type):
|
||||
err = reflection.get_class_name(cls)
|
||||
else:
|
||||
err = cls
|
||||
if err in self._exc_type_names:
|
||||
return cls
|
||||
return None
|
||||
|
||||
def __str__(self):
|
||||
return self.pformat()
|
||||
|
||||
def pformat(self, traceback=False):
|
||||
"""Pretty formats the failure object into a string."""
|
||||
buf = six.StringIO()
|
||||
buf.write(
|
||||
'Failure: %s: %s' % (self._exc_type_names[0], self._exception_str))
|
||||
if traceback:
|
||||
if self._traceback_str is not None:
|
||||
traceback_str = self._traceback_str.rstrip()
|
||||
else:
|
||||
traceback_str = None
|
||||
if traceback_str:
|
||||
buf.write('\nTraceback (most recent call last):\n')
|
||||
buf.write(traceback_str)
|
||||
else:
|
||||
buf.write('\nTraceback not available.')
|
||||
return buf.getvalue()
|
||||
|
||||
def __iter__(self):
|
||||
"""Iterate over exception type names."""
|
||||
for et in self._exc_type_names:
|
||||
yield et
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data):
|
||||
"""Converts this from a dictionary to a object."""
|
||||
data = dict(data)
|
||||
version = data.pop('version', None)
|
||||
if version != cls.DICT_VERSION:
|
||||
raise ValueError('Invalid dict version of failure object: %r'
|
||||
% version)
|
||||
return cls(**data)
|
||||
|
||||
def to_dict(self):
|
||||
"""Converts this object to a dictionary."""
|
||||
return {
|
||||
'exception_str': self.exception_str,
|
||||
'traceback_str': self.traceback_str,
|
||||
'exc_type_names': list(self),
|
||||
'version': self.DICT_VERSION,
|
||||
}
|
||||
|
||||
def copy(self):
|
||||
"""Copies this object."""
|
||||
return Failure(exc_info=copy_exc_info(self.exc_info),
|
||||
exception_str=self.exception_str,
|
||||
traceback_str=self.traceback_str,
|
||||
exc_type_names=self._exc_type_names[:])
|
||||
|
||||
Reference in New Issue
Block a user