From 3abefc717f2788e604a3fe2aa3de442d2156a1ee Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 16 Nov 2010 22:47:56 +0000 Subject: [PATCH 01/49] Use plain raise in Raises.match which is wrong on Python 2 but untested --- testtools/matchers.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/testtools/matchers.py b/testtools/matchers.py index 75ea01f..8c92107 100644 --- a/testtools/matchers.py +++ b/testtools/matchers.py @@ -467,7 +467,6 @@ class Raises(Matcher): # Catch all exceptions: Raises() should be able to match a # KeyboardInterrupt or SystemExit. except: - exc_info = sys.exc_info() if self.exception_matcher: mismatch = self.exception_matcher.match(sys.exc_info()) if not mismatch: @@ -477,8 +476,8 @@ class Raises(Matcher): # The exception did not match, or no explicit matching logic was # performed. If the exception is a non-user exception (that is, not # a subclass of Exception) then propogate it. - if not issubclass(exc_info[0], Exception): - raise exc_info[0], exc_info[1], exc_info[2] + if not issubclass(sys.exc_info()[0], Exception): + raise return mismatch def __str__(self): From becd91988c598a17610fa9a791103ccd442119be Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 16 Nov 2010 23:04:47 +0000 Subject: [PATCH 02/49] Weaken MatchesException stringification tests so they don't break on Exception implementation variations --- testtools/tests/test_matchers.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/testtools/tests/test_matchers.py b/testtools/tests/test_matchers.py index 20360ba..2958cb5 100644 --- a/testtools/tests/test_matchers.py +++ b/testtools/tests/test_matchers.py @@ -181,8 +181,7 @@ class TestMatchesExceptionInstanceInterface(TestCase, TestMatchersInterface): MatchesException(Exception('foo'))) ] describe_examples = [ - (" is not a " - "", + ("%r is not a %r" % (Exception, ValueError), error_base_foo, MatchesException(ValueError("foo"))), ("ValueError('bar',) has different arguments to ValueError('foo',).", @@ -201,12 +200,11 @@ class TestMatchesExceptionTypeInterface(TestCase, TestMatchersInterface): matches_mismatches = [error_base_foo] str_examples = [ - ("MatchesException()", + ("MatchesException(%r)" % (Exception,), MatchesException(Exception)) ] describe_examples = [ - (" is not a " - "", + ("%r is not a %r" % (Exception, ValueError), error_base_foo, MatchesException(ValueError)), ] From b2b223185fd02010d106a53467af0aef9cbf885d Mon Sep 17 00:00:00 2001 From: Martin Date: Tue, 16 Nov 2010 23:58:30 +0000 Subject: [PATCH 03/49] Adapt MatchesException to work with old-style Exception classes in Python 2.4 --- testtools/matchers.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/testtools/matchers.py b/testtools/matchers.py index 8c92107..2845494 100644 --- a/testtools/matchers.py +++ b/testtools/matchers.py @@ -33,6 +33,8 @@ import operator from pprint import pformat import sys +from testtools.compat import classtypes + class Matcher(object): """A pattern matcher. @@ -346,9 +348,9 @@ class MatchesException(Matcher): self.expected = exception def _expected_type(self): - if type(self.expected) is type: + if type(self.expected) in classtypes(): return self.expected - return type(self.expected) + return self.expected.__class__ def match(self, other): if type(other) != tuple: @@ -356,7 +358,7 @@ class MatchesException(Matcher): if not issubclass(other[0], self._expected_type()): return Mismatch('%r is not a %r' % ( other[0], self._expected_type())) - if (type(self.expected) is not type and + if (type(self.expected) not in classtypes() and other[1].args != self.expected.args): return Mismatch('%r has different arguments to %r.' % ( other[1], self.expected)) From e5ac4755b69cdad1791fbdce8a43113c96e4df1d Mon Sep 17 00:00:00 2001 From: Martin Date: Fri, 19 Nov 2010 16:33:35 +0000 Subject: [PATCH 04/49] Rearrange MatchesException code and change str representation to work better with Python 2.4 --- testtools/compat.py | 8 ++++++++ testtools/matchers.py | 27 +++++++++++++-------------- testtools/tests/test_matchers.py | 2 +- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/testtools/compat.py b/testtools/compat.py index 1f0b8cf..c1e734c 100644 --- a/testtools/compat.py +++ b/testtools/compat.py @@ -65,6 +65,14 @@ else: _u.__doc__ = __u_doc +if sys.version_info > (2, 5): + _error_repr = BaseException.__repr__ +else: + def _error_repr(exception): + """Format an exception instance as Python 2.5 and later do""" + return exception.__class__.__name__ + repr(exception.args) + + def unicode_output_stream(stream): """Get wrapper for given stream that writes any unicode without exception diff --git a/testtools/matchers.py b/testtools/matchers.py index 2845494..d404822 100644 --- a/testtools/matchers.py +++ b/testtools/matchers.py @@ -33,7 +33,7 @@ import operator from pprint import pformat import sys -from testtools.compat import classtypes +from testtools.compat import classtypes, _error_repr class Matcher(object): @@ -346,25 +346,24 @@ class MatchesException(Matcher): """ Matcher.__init__(self) self.expected = exception - - def _expected_type(self): - if type(self.expected) in classtypes(): - return self.expected - return self.expected.__class__ + self._is_instance = type(self.expected) not in classtypes() def match(self, other): if type(other) != tuple: return Mismatch('%r is not an exc_info tuple' % other) - if not issubclass(other[0], self._expected_type()): - return Mismatch('%r is not a %r' % ( - other[0], self._expected_type())) - if (type(self.expected) not in classtypes() and - other[1].args != self.expected.args): - return Mismatch('%r has different arguments to %r.' % ( - other[1], self.expected)) + expected_class = self.expected + if self._is_instance: + expected_class = expected_class.__class__ + if not issubclass(other[0], expected_class): + return Mismatch('%r is not a %r' % (other[0], expected_class)) + if self._is_instance and other[1].args != self.expected.args: + return Mismatch('%s has different arguments to %s.' % ( + _error_repr(other[1]), _error_repr(self.expected))) def __str__(self): - return "MatchesException(%r)" % self.expected + if self._is_instance: + return "MatchesException(%s)" % _error_repr(self.expected) + return "MatchesException(%s)" % self.expected.__name__ class StartsWith(Matcher): diff --git a/testtools/tests/test_matchers.py b/testtools/tests/test_matchers.py index 2958cb5..845e0a5 100644 --- a/testtools/tests/test_matchers.py +++ b/testtools/tests/test_matchers.py @@ -200,7 +200,7 @@ class TestMatchesExceptionTypeInterface(TestCase, TestMatchersInterface): matches_mismatches = [error_base_foo] str_examples = [ - ("MatchesException(%r)" % (Exception,), + ("MatchesException(Exception)", MatchesException(Exception)) ] describe_examples = [ From 1d2d71e732ffa2655e226635e8948c0881583ba6 Mon Sep 17 00:00:00 2001 From: Martin Date: Fri, 19 Nov 2010 18:11:39 +0000 Subject: [PATCH 05/49] Implement isbaseexception to resolve failures from exception hierarchy changes in Python 2.5 --- testtools/compat.py | 13 +++++++++++++ testtools/matchers.py | 6 +++--- testtools/tests/test_matchers.py | 7 ++++++- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/testtools/compat.py b/testtools/compat.py index c1e734c..5c2b224 100644 --- a/testtools/compat.py +++ b/testtools/compat.py @@ -67,10 +67,23 @@ _u.__doc__ = __u_doc if sys.version_info > (2, 5): _error_repr = BaseException.__repr__ + def isbaseexception(exception): + """Return whether exception inherits from BaseException only""" + return (isinstance(exception, BaseException) + and not isinstance(exception, Exception)) else: def _error_repr(exception): """Format an exception instance as Python 2.5 and later do""" return exception.__class__.__name__ + repr(exception.args) + def isbaseexception(exception): + """Return whether exception would inherit from BaseException only + + This approximates the hierarchy in Python 2.5 and later, compare the + difference between the diagrams at the bottom of the pages: + + + """ + return isinstance(exception, (KeyboardInterrupt, SystemExit)) def unicode_output_stream(stream): diff --git a/testtools/matchers.py b/testtools/matchers.py index d404822..0664ac9 100644 --- a/testtools/matchers.py +++ b/testtools/matchers.py @@ -33,7 +33,7 @@ import operator from pprint import pformat import sys -from testtools.compat import classtypes, _error_repr +from testtools.compat import classtypes, _error_repr, isbaseexception class Matcher(object): @@ -476,8 +476,8 @@ class Raises(Matcher): mismatch = None # The exception did not match, or no explicit matching logic was # performed. If the exception is a non-user exception (that is, not - # a subclass of Exception) then propogate it. - if not issubclass(sys.exc_info()[0], Exception): + # a subclass of Exception on Python 2.5+) then propogate it. + if isbaseexception(sys.exc_info()[1]): raise return mismatch diff --git a/testtools/tests/test_matchers.py b/testtools/tests/test_matchers.py index 845e0a5..f0253a8 100644 --- a/testtools/tests/test_matchers.py +++ b/testtools/tests/test_matchers.py @@ -360,7 +360,12 @@ class TestRaisesBaseTypes(TestCase): # Exception, it is propogated. match_keyb = Raises(MatchesException(KeyboardInterrupt)) def raise_keyb_from_match(): - matcher = Raises(MatchesException(Exception)) + if sys.version_info > (2, 5): + matcher = Raises(MatchesException(Exception)) + else: + # On Python 2.4 KeyboardInterrupt is a StandardError subclass + # but should propogate from less generic exception matchers + matcher = Raises(MatchesException(EnvironmentError)) matcher.match(self.raiser) self.assertThat(raise_keyb_from_match, match_keyb) From 87e76b34360e99dd7e05bf31a839697a087d54cd Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Sat, 27 Nov 2010 10:51:14 +0000 Subject: [PATCH 06/49] Move responsibility for running cleanups to RunTest. --- NEWS | 4 ++++ testtools/__init__.py | 2 +- testtools/runtest.py | 43 ++++++++++++++++++++++++++++++++++++++++--- testtools/testcase.py | 37 ------------------------------------- 4 files changed, 45 insertions(+), 41 deletions(-) diff --git a/NEWS b/NEWS index fb46a4d..eade01d 100644 --- a/NEWS +++ b/NEWS @@ -10,6 +10,10 @@ Changes * addUnexpectedSuccess is translated to addFailure for test results that don't know about addUnexpectedSuccess. (Jonathan Lange, #654474) +* Responsibility for running test cleanups has been moved to ``RunTest``. + This change does not affect public APIs and can be safely ignored by test + authors. (Jonathan Lange, #662647) + Improvements ------------ diff --git a/testtools/__init__.py b/testtools/__init__.py index c3f6701..0f85426 100644 --- a/testtools/__init__.py +++ b/testtools/__init__.py @@ -32,11 +32,11 @@ from testtools.matchers import ( Matcher, ) from testtools.runtest import ( + MultipleExceptions, RunTest, ) from testtools.testcase import ( ErrorHolder, - MultipleExceptions, PlaceHolder, TestCase, clone_test_with_new_id, diff --git a/testtools/runtest.py b/testtools/runtest.py index 3adabf0..e56512b 100644 --- a/testtools/runtest.py +++ b/testtools/runtest.py @@ -1,8 +1,9 @@ -# Copyright (c) 2009 Jonathan M. Lange. See LICENSE for details. +# Copyright (c) 2009-2010 Jonathan M. Lange. See LICENSE for details. """Individual test case execution.""" __all__ = [ + 'MultipleExceptions', 'RunTest', ] @@ -11,6 +12,13 @@ import sys from testtools.testresult import ExtendedToOriginalDecorator +class MultipleExceptions(Exception): + """Represents many exceptions raised from some operation. + + :ivar args: The sys.exc_info() tuples for each exception. + """ + + class RunTest(object): """An object to run a test. @@ -107,7 +115,7 @@ class RunTest(object): if self.exception_caught == self._run_user(self.case._run_setup, self.result): # Don't run the test method if we failed getting here. - e = self.case._runCleanups(self.result) + e = self._run_cleanups(self.result) if e is not None: self._exceptions.append(e) return @@ -125,7 +133,7 @@ class RunTest(object): failed = True finally: try: - e = self._run_user(self.case._runCleanups, self.result) + e = self._run_user(self._run_cleanups, self.result) if e is not None: self._exceptions.append(e) failed = True @@ -146,6 +154,35 @@ class RunTest(object): except: return self._got_user_exception(sys.exc_info()) + def _run_cleanups(self, result): + """Run the cleanups that have been added with addCleanup. + + See the docstring for addCleanup for more information. + + :return: None if all cleanups ran without error, the most recently + raised exception from the cleanups otherwise. + """ + last_exception = None + while self.case._cleanups: + function, arguments, keywordArguments = self.case._cleanups.pop() + try: + function(*arguments, **keywordArguments) + except KeyboardInterrupt: + raise + except: + exceptions = [sys.exc_info()] + while exceptions: + try: + exc_info = exceptions.pop() + if exc_info[0] is MultipleExceptions: + exceptions.extend(exc_info[1].args) + continue + self.case._report_traceback(exc_info) + last_exception = exc_info[1] + finally: + del exc_info + return last_exception + def _got_user_exception(self, exc_info, tb_label='traceback'): """Called when user code raises an exception. diff --git a/testtools/testcase.py b/testtools/testcase.py index 58e145c..ba7b480 100644 --- a/testtools/testcase.py +++ b/testtools/testcase.py @@ -5,7 +5,6 @@ __metaclass__ = type __all__ = [ 'clone_test_with_new_id', - 'MultipleExceptions', 'run_test_with', 'skip', 'skipIf', @@ -92,13 +91,6 @@ def run_test_with(test_runner, **kwargs): return decorator -class MultipleExceptions(Exception): - """Represents many exceptions raised from some operation. - - :ivar args: The sys.exc_info() tuples for each exception. - """ - - class TestCase(unittest.TestCase): """Extensions to the basic TestCase. @@ -224,35 +216,6 @@ class TestCase(unittest.TestCase): className = ', '.join(klass.__name__ for klass in classOrIterable) return className - def _runCleanups(self, result): - """Run the cleanups that have been added with addCleanup. - - See the docstring for addCleanup for more information. - - :return: None if all cleanups ran without error, the most recently - raised exception from the cleanups otherwise. - """ - last_exception = None - while self._cleanups: - function, arguments, keywordArguments = self._cleanups.pop() - try: - function(*arguments, **keywordArguments) - except KeyboardInterrupt: - raise - except: - exceptions = [sys.exc_info()] - while exceptions: - try: - exc_info = exceptions.pop() - if exc_info[0] is MultipleExceptions: - exceptions.extend(exc_info[1].args) - continue - self._report_traceback(exc_info) - last_exception = exc_info[1] - finally: - del exc_info - return last_exception - def addCleanup(self, function, *arguments, **keywordArguments): """Add a cleanup function to be called after tearDown. From 177bf8cbd7765092a4d4f5073a76a70c463da842 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Sat, 27 Nov 2010 11:14:36 +0000 Subject: [PATCH 07/49] Skip the spinner tests, rather than elide them. --- testtools/tests/__init__.py | 5 +- testtools/tests/test_spinner.py | 99 +++++++++++++++++---------------- 2 files changed, 54 insertions(+), 50 deletions(-) diff --git a/testtools/tests/__init__.py b/testtools/tests/__init__.py index d00ccea..d8a6feb 100644 --- a/testtools/tests/__init__.py +++ b/testtools/tests/__init__.py @@ -14,6 +14,7 @@ def test_suite(): test_matchers, test_monkey, test_runtest, + test_spinner, test_testtools, test_testresult, test_testsuite, @@ -27,6 +28,7 @@ def test_suite(): test_matchers, test_monkey, test_runtest, + test_spinner, test_testresult, test_testsuite, test_testtools, @@ -35,12 +37,11 @@ def test_suite(): # Tests that rely on Twisted. from testtools.tests import ( test_deferredruntest, - test_spinner, ) except ImportError: pass else: - modules.extend([test_deferredruntest, test_spinner]) + modules.extend([test_deferredruntest]) try: # Tests that rely on 'fixtures'. from testtools.tests import ( diff --git a/testtools/tests/test_spinner.py b/testtools/tests/test_spinner.py index b235a1c..f898956 100644 --- a/testtools/tests/test_spinner.py +++ b/testtools/tests/test_spinner.py @@ -9,90 +9,91 @@ from testtools import ( skipIf, TestCase, ) +from testtools.helpers import try_import from testtools.matchers import ( Equals, Is, MatchesException, Raises, ) -from testtools._spinner import ( - DeferredNotFired, - extract_result, - NoResultError, - not_reentrant, - ReentryError, - Spinner, - StaleJunkError, - TimeoutError, - trap_unhandled_errors, - ) -from twisted.internet import defer -from twisted.python.failure import Failure +_spinner = try_import('testtools._spinner') + +defer = try_import('twisted.internet.defer') +Failure = try_import('twisted.python.failure.Failure') -class TestNotReentrant(TestCase): +class NeedsTwistedTestCase(TestCase): + + def setUp(self): + super(NeedsTwistedTestCase, self).setUp() + if defer is None or Failure is None: + self.skipTest("Need Twisted to run") + + +class TestNotReentrant(NeedsTwistedTestCase): def test_not_reentrant(self): # A function decorated as not being re-entrant will raise a - # ReentryError if it is called while it is running. + # _spinner.ReentryError if it is called while it is running. calls = [] - @not_reentrant + @_spinner.not_reentrant def log_something(): calls.append(None) if len(calls) < 5: log_something() - self.assertThat(log_something, Raises(MatchesException(ReentryError))) + self.assertThat( + log_something, Raises(MatchesException(_spinner.ReentryError))) self.assertEqual(1, len(calls)) def test_deeper_stack(self): calls = [] - @not_reentrant + @_spinner.not_reentrant def g(): calls.append(None) if len(calls) < 5: f() - @not_reentrant + @_spinner.not_reentrant def f(): calls.append(None) if len(calls) < 5: g() - self.assertThat(f, Raises(MatchesException(ReentryError))) + self.assertThat(f, Raises(MatchesException(_spinner.ReentryError))) self.assertEqual(2, len(calls)) -class TestExtractResult(TestCase): +class TestExtractResult(NeedsTwistedTestCase): def test_not_fired(self): - # extract_result raises DeferredNotFired if it's given a Deferred that - # has not fired. - self.assertThat(lambda:extract_result(defer.Deferred()), - Raises(MatchesException(DeferredNotFired))) + # _spinner.extract_result raises _spinner.DeferredNotFired if it's + # given a Deferred that has not fired. + self.assertThat(lambda:_spinner.extract_result(defer.Deferred()), + Raises(MatchesException(_spinner.DeferredNotFired))) def test_success(self): - # extract_result returns the value of the Deferred if it has fired - # successfully. + # _spinner.extract_result returns the value of the Deferred if it has + # fired successfully. marker = object() d = defer.succeed(marker) - self.assertThat(extract_result(d), Equals(marker)) + self.assertThat(_spinner.extract_result(d), Equals(marker)) def test_failure(self): - # extract_result raises the failure's exception if it's given a - # Deferred that is failing. + # _spinner.extract_result raises the failure's exception if it's given + # a Deferred that is failing. try: 1/0 except ZeroDivisionError: f = Failure() d = defer.fail(f) - self.assertThat(lambda:extract_result(d), + self.assertThat(lambda:_spinner.extract_result(d), Raises(MatchesException(ZeroDivisionError))) -class TestTrapUnhandledErrors(TestCase): +class TestTrapUnhandledErrors(NeedsTwistedTestCase): def test_no_deferreds(self): marker = object() - result, errors = trap_unhandled_errors(lambda: marker) + result, errors = _spinner.trap_unhandled_errors(lambda: marker) self.assertEqual([], errors) self.assertIs(marker, result) @@ -105,12 +106,13 @@ class TestTrapUnhandledErrors(TestCase): f = Failure() failures.append(f) defer.fail(f) - result, errors = trap_unhandled_errors(make_deferred_but_dont_handle) + result, errors = _spinner.trap_unhandled_errors( + make_deferred_but_dont_handle) self.assertIs(None, result) self.assertEqual(failures, [error.failResult for error in errors]) -class TestRunInReactor(TestCase): +class TestRunInReactor(NeedsTwistedTestCase): def make_reactor(self): from twisted.internet import reactor @@ -119,7 +121,7 @@ class TestRunInReactor(TestCase): def make_spinner(self, reactor=None): if reactor is None: reactor = self.make_reactor() - return Spinner(reactor) + return _spinner.Spinner(reactor) def make_timeout(self): return 0.01 @@ -157,8 +159,8 @@ class TestRunInReactor(TestCase): # to run_in_reactor. spinner = self.make_spinner() self.assertThat(lambda: spinner.run( - self.make_timeout(), spinner.run, self.make_timeout(), lambda: None), - Raises(MatchesException(ReentryError))) + self.make_timeout(), spinner.run, self.make_timeout(), + lambda: None), Raises(MatchesException(_spinner.ReentryError))) def test_deferred_value_returned(self): # If the given function returns a Deferred, run_in_reactor returns the @@ -182,11 +184,12 @@ class TestRunInReactor(TestCase): self.assertEqual(new_hdlrs, map(signal.getsignal, signals)) def test_timeout(self): - # If the function takes too long to run, we raise a TimeoutError. + # If the function takes too long to run, we raise a + # _spinner.TimeoutError. timeout = self.make_timeout() self.assertThat( lambda:self.make_spinner().run(timeout, lambda: defer.Deferred()), - Raises(MatchesException(TimeoutError))) + Raises(MatchesException(_spinner.TimeoutError))) def test_no_junk_by_default(self): # If the reactor hasn't spun yet, then there cannot be any junk. @@ -263,7 +266,7 @@ class TestRunInReactor(TestCase): timeout = self.make_timeout() spinner.run(timeout, reactor.listenTCP, 0, ServerFactory()) self.assertThat(lambda: spinner.run(timeout, lambda: None), - Raises(MatchesException(StaleJunkError))) + Raises(MatchesException(_spinner.StaleJunkError))) def test_clear_junk_clears_previous_junk(self): # If 'run' is called and there's still junk in the spinner's junk @@ -279,7 +282,7 @@ class TestRunInReactor(TestCase): @skipIf(os.name != "posix", "Sending SIGINT with os.kill is posix only") def test_sigint_raises_no_result_error(self): - # If we get a SIGINT during a run, we raise NoResultError. + # If we get a SIGINT during a run, we raise _spinner.NoResultError. SIGINT = getattr(signal, 'SIGINT', None) if not SIGINT: self.skipTest("SIGINT not available") @@ -288,19 +291,19 @@ class TestRunInReactor(TestCase): timeout = self.make_timeout() reactor.callLater(timeout, os.kill, os.getpid(), SIGINT) self.assertThat(lambda:spinner.run(timeout * 5, defer.Deferred), - Raises(MatchesException(NoResultError))) + Raises(MatchesException(_spinner.NoResultError))) self.assertEqual([], spinner._clean()) @skipIf(os.name != "posix", "Sending SIGINT with os.kill is posix only") def test_sigint_raises_no_result_error_second_time(self): - # If we get a SIGINT during a run, we raise NoResultError. This test - # is exactly the same as test_sigint_raises_no_result_error, and - # exists to make sure we haven't futzed with state. + # If we get a SIGINT during a run, we raise _spinner.NoResultError. + # This test is exactly the same as test_sigint_raises_no_result_error, + # and exists to make sure we haven't futzed with state. self.test_sigint_raises_no_result_error() @skipIf(os.name != "posix", "Sending SIGINT with os.kill is posix only") def test_fast_sigint_raises_no_result_error(self): - # If we get a SIGINT during a run, we raise NoResultError. + # If we get a SIGINT during a run, we raise _spinner.NoResultError. SIGINT = getattr(signal, 'SIGINT', None) if not SIGINT: self.skipTest("SIGINT not available") @@ -309,7 +312,7 @@ class TestRunInReactor(TestCase): timeout = self.make_timeout() reactor.callWhenRunning(os.kill, os.getpid(), SIGINT) self.assertThat(lambda:spinner.run(timeout * 5, defer.Deferred), - Raises(MatchesException(NoResultError))) + Raises(MatchesException(_spinner.NoResultError))) self.assertEqual([], spinner._clean()) @skipIf(os.name != "posix", "Sending SIGINT with os.kill is posix only") From 25c71f3b2e3ea9043855afd1cfbe706955c3fb9d Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Sat, 27 Nov 2010 11:22:15 +0000 Subject: [PATCH 08/49] Always run the deferred run tests. --- testtools/tests/__init__.py | 11 ++----- testtools/tests/test_deferredruntest.py | 42 ++++++++++++++----------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/testtools/tests/__init__.py b/testtools/tests/__init__.py index d8a6feb..2427952 100644 --- a/testtools/tests/__init__.py +++ b/testtools/tests/__init__.py @@ -10,6 +10,7 @@ def test_suite(): test_compat, test_content, test_content_type, + test_deferredruntest, test_helpers, test_matchers, test_monkey, @@ -24,6 +25,7 @@ def test_suite(): test_compat, test_content, test_content_type, + test_deferredruntest, test_helpers, test_matchers, test_monkey, @@ -33,15 +35,6 @@ def test_suite(): test_testsuite, test_testtools, ] - try: - # Tests that rely on Twisted. - from testtools.tests import ( - test_deferredruntest, - ) - except ImportError: - pass - else: - modules.extend([test_deferredruntest]) try: # Tests that rely on 'fixtures'. from testtools.tests import ( diff --git a/testtools/tests/test_deferredruntest.py b/testtools/tests/test_deferredruntest.py index a2ffdaa..c56cccf 100644 --- a/testtools/tests/test_deferredruntest.py +++ b/testtools/tests/test_deferredruntest.py @@ -12,12 +12,7 @@ from testtools import ( from testtools.content import ( text_content, ) -from testtools.deferredruntest import ( - assert_fails_with, - AsynchronousDeferredRunTest, - flush_logged_errors, - SynchronousDeferredRunTest, - ) +from testtools.helpers import try_import from testtools.tests.helpers import ExtendedTestResult from testtools.matchers import ( Equals, @@ -26,9 +21,19 @@ from testtools.matchers import ( Raises, ) from testtools.runtest import RunTest +from testtools.tests.test_spinner import NeedsTwistedTestCase -from twisted.internet import defer -from twisted.python import failure, log +assert_fails_with = try_import('testtools.deferredruntest.assert_fails_with') +AsynchronousDeferredRunTest = try_import( + 'testtools.deferredruntest.AsynchronousDeferredRunTest') +flush_logged_errors = try_import( + 'testtools.deferredruntest.flush_logged_errors') +SynchronousDeferredRunTest = try_import( + 'testtools.deferredruntest.SynchronousDeferredRunTest') + +defer = try_import('twisted.internet.defer') +failure = try_import('twisted.python.failure') +log = try_import('twisted.python.log') class X(object): @@ -77,7 +82,7 @@ class X(object): self.calls.append('test') self.addCleanup(lambda: 1/0) - class TestIntegration(TestCase): + class TestIntegration(NeedsTwistedTestCase): def assertResultsMatch(self, test, result): events = list(result._events) @@ -104,9 +109,9 @@ def make_integration_tests(): from unittest import TestSuite from testtools import clone_test_with_new_id runners = [ - RunTest, - SynchronousDeferredRunTest, - AsynchronousDeferredRunTest, + ('RunTest', RunTest), + ('SynchronousDeferredRunTest', SynchronousDeferredRunTest), + ('AsynchronousDeferredRunTest', AsynchronousDeferredRunTest), ] tests = [ @@ -118,12 +123,12 @@ def make_integration_tests(): ] base_test = X.TestIntegration('test_runner') integration_tests = [] - for runner in runners: + for runner_name, runner in runners: for test in tests: new_test = clone_test_with_new_id( base_test, '%s(%s, %s)' % ( base_test.id(), - runner.__name__, + runner_name, test.__name__)) new_test.test_factory = test new_test.runner = runner @@ -131,7 +136,7 @@ def make_integration_tests(): return TestSuite(integration_tests) -class TestSynchronousDeferredRunTest(TestCase): +class TestSynchronousDeferredRunTest(NeedsTwistedTestCase): def make_result(self): return ExtendedTestResult() @@ -185,7 +190,7 @@ class TestSynchronousDeferredRunTest(TestCase): ('stopTest', test)])) -class TestAsynchronousDeferredRunTest(TestCase): +class TestAsynchronousDeferredRunTest(NeedsTwistedTestCase): def make_reactor(self): from twisted.internet import reactor @@ -602,10 +607,11 @@ class TestAsynchronousDeferredRunTest(TestCase): self.assertThat(error, KeysEqual('traceback', 'twisted-log')) -class TestAssertFailsWith(TestCase): +class TestAssertFailsWith(NeedsTwistedTestCase): """Tests for `assert_fails_with`.""" - run_tests_with = SynchronousDeferredRunTest + if SynchronousDeferredRunTest is not None: + run_tests_with = SynchronousDeferredRunTest def test_assert_fails_with_success(self): # assert_fails_with fails the test if it's given a Deferred that From 65e2df185768862b8fa8671eeae15732c13488c7 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Sat, 27 Nov 2010 11:25:12 +0000 Subject: [PATCH 09/49] Do the same for fixtures. --- testtools/tests/__init__.py | 12 ++---------- testtools/tests/test_fixturesupport.py | 12 +++++++++--- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/testtools/tests/__init__.py b/testtools/tests/__init__.py index 2427952..b3b9b4e 100644 --- a/testtools/tests/__init__.py +++ b/testtools/tests/__init__.py @@ -11,6 +11,7 @@ def test_suite(): test_content, test_content_type, test_deferredruntest, + test_fixturesupport, test_helpers, test_matchers, test_monkey, @@ -26,6 +27,7 @@ def test_suite(): test_content, test_content_type, test_deferredruntest, + test_fixturesupport, test_helpers, test_matchers, test_monkey, @@ -35,16 +37,6 @@ def test_suite(): test_testsuite, test_testtools, ] - try: - # Tests that rely on 'fixtures'. - from testtools.tests import ( - test_fixturesupport, - ) - except ImportError: - pass - else: - modules.extend([test_fixturesupport]) - for module in modules: suites.append(getattr(module, 'test_suite')()) return unittest.TestSuite(suites) diff --git a/testtools/tests/test_fixturesupport.py b/testtools/tests/test_fixturesupport.py index c9ca154..ebdd037 100644 --- a/testtools/tests/test_fixturesupport.py +++ b/testtools/tests/test_fixturesupport.py @@ -1,20 +1,26 @@ import unittest -import fixtures -from fixtures.tests.helpers import LoggingFixture - from testtools import ( TestCase, content, content_type, ) +from testtools.helpers import try_import from testtools.tests.helpers import ( ExtendedTestResult, ) +fixtures = try_import('fixtures') +LoggingFixture = try_import('fixtures.tests.helpers.LoggingFixture') + class TestFixtureSupport(TestCase): + def setUp(self): + super(TestFixtureSupport, self).setUp() + if fixtures is None or LoggingFixture is None: + self.skipTest("Need fixtures") + def test_useFixture(self): fixture = LoggingFixture() class SimpleTest(TestCase): From 564b1026cb5d8473593bacc49279dcb56ded9cad Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Sat, 27 Nov 2010 11:37:06 +0000 Subject: [PATCH 10/49] EndsWith matcher. --- NEWS | 3 +++ testtools/matchers.py | 35 ++++++++++++++++++++++++++++++++ testtools/tests/test_matchers.py | 34 +++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/NEWS b/NEWS index fb46a4d..bbd725e 100644 --- a/NEWS +++ b/NEWS @@ -22,6 +22,9 @@ Improvements * Fix the runTest parameter of TestCase to actually work, rather than raising a TypeError. (Jonathan Lange, #657760) +* New matcher ``EndsWith`` added to complement the existing ``StartsWith`` + matcher. (Jonathan Lange, #669165) + * Non-release snapshots of testtools will now work with buildout. (Jonathan Lange, #613734) diff --git a/testtools/matchers.py b/testtools/matchers.py index 75ea01f..8b8f442 100644 --- a/testtools/matchers.py +++ b/testtools/matchers.py @@ -179,6 +179,22 @@ class DoesNotStartWith(Mismatch): self.matchee, self.expected) +class DoesNotEndWith(Mismatch): + + def __init__(self, matchee, expected): + """Create a DoesNotEndWith Mismatch. + + :param matchee: the string that did not match. + :param expected: the string that `matchee` was expected to end with. + """ + self.matchee = matchee + self.expected = expected + + def describe(self): + return "'%s' does not end with '%s'." % ( + self.matchee, self.expected) + + class _BinaryComparison(object): """Matcher that compares an object to another object.""" @@ -384,6 +400,25 @@ class StartsWith(Matcher): return None +class EndsWith(Matcher): + """Checks whether one string starts with another.""" + + def __init__(self, expected): + """Create a EndsWith Matcher. + + :param expected: the string that matchees should end with. + """ + self.expected = expected + + def __str__(self): + return "Ends with '%s'." % self.expected + + def match(self, matchee): + if not matchee.endswith(self.expected): + return DoesNotEndWith(matchee, self.expected) + return None + + class KeysEqual(Matcher): """Checks whether a dict has particular keys.""" diff --git a/testtools/tests/test_matchers.py b/testtools/tests/test_matchers.py index 20360ba..9cc2c01 100644 --- a/testtools/tests/test_matchers.py +++ b/testtools/tests/test_matchers.py @@ -13,7 +13,9 @@ from testtools.matchers import ( Annotate, Equals, DocTestMatches, + DoesNotEndWith, DoesNotStartWith, + EndsWith, KeysEqual, Is, LessThan, @@ -411,6 +413,38 @@ class StartsWithTests(TestCase): self.assertEqual("bar", mismatch.expected) +class DoesNotEndWithTests(TestCase): + + def test_describe(self): + mismatch = DoesNotEndWith("fo", "bo") + self.assertEqual("'fo' does not end with 'bo'.", mismatch.describe()) + + +class EndsWithTests(TestCase): + + def test_str(self): + matcher = EndsWith("bar") + self.assertEqual("Ends with 'bar'.", str(matcher)) + + def test_match(self): + matcher = EndsWith("arf") + self.assertIs(None, matcher.match("barf")) + + def test_mismatch_returns_does_not_end_with(self): + matcher = EndsWith("bar") + self.assertIsInstance(matcher.match("foo"), DoesNotEndWith) + + def test_mismatch_sets_matchee(self): + matcher = EndsWith("bar") + mismatch = matcher.match("foo") + self.assertEqual("foo", mismatch.matchee) + + def test_mismatch_sets_expected(self): + matcher = EndsWith("bar") + mismatch = matcher.match("foo") + self.assertEqual("bar", mismatch.expected) + + def test_suite(): from unittest import TestLoader return TestLoader().loadTestsFromName(__name__) From b7937ec1a5820ab2888f669a7d3cb6f862dac966 Mon Sep 17 00:00:00 2001 From: Martin Date: Sat, 27 Nov 2010 21:26:32 +0000 Subject: [PATCH 11/49] Go back to using repr for exception types in MatchesException.__str__ --- testtools/matchers.py | 2 +- testtools/tests/test_matchers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testtools/matchers.py b/testtools/matchers.py index 0664ac9..9c6684a 100644 --- a/testtools/matchers.py +++ b/testtools/matchers.py @@ -363,7 +363,7 @@ class MatchesException(Matcher): def __str__(self): if self._is_instance: return "MatchesException(%s)" % _error_repr(self.expected) - return "MatchesException(%s)" % self.expected.__name__ + return "MatchesException(%s)" % repr(self.expected) class StartsWith(Matcher): diff --git a/testtools/tests/test_matchers.py b/testtools/tests/test_matchers.py index f0253a8..23d9bfb 100644 --- a/testtools/tests/test_matchers.py +++ b/testtools/tests/test_matchers.py @@ -200,7 +200,7 @@ class TestMatchesExceptionTypeInterface(TestCase, TestMatchersInterface): matches_mismatches = [error_base_foo] str_examples = [ - ("MatchesException(Exception)", + ("MatchesException(%r)" % Exception, MatchesException(Exception)) ] describe_examples = [ From 47d9a6d3a47739ce868518f7d5338f043d9e4135 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Sun, 28 Nov 2010 12:15:49 +0000 Subject: [PATCH 12/49] Pass through kwargs. --- testtools/runtest.py | 4 ++-- testtools/tests/test_testresult.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/testtools/runtest.py b/testtools/runtest.py index e56512b..ae843a7 100644 --- a/testtools/runtest.py +++ b/testtools/runtest.py @@ -142,13 +142,13 @@ class RunTest(object): self.result.addSuccess(self.case, details=self.case.getDetails()) - def _run_user(self, fn, *args): + def _run_user(self, fn, *args, **kwargs): """Run a user supplied function. Exceptions are processed by self.handlers. """ try: - return fn(*args) + return fn(*args, **kwargs) except KeyboardInterrupt: raise except: diff --git a/testtools/tests/test_testresult.py b/testtools/tests/test_testresult.py index 5686b7f..ad466ec 100644 --- a/testtools/tests/test_testresult.py +++ b/testtools/tests/test_testresult.py @@ -407,7 +407,7 @@ Text attachment: traceback ------------ Traceback (most recent call last): File "...testtools...runtest.py", line ..., in _run_user... - return fn(*args) + return fn(*args, **kwargs) File "...testtools...testcase.py", line ..., in _run_test_method return self._get_test_method()() File "...testtools...tests...test_testresult.py", line ..., in error @@ -421,7 +421,7 @@ Text attachment: traceback ------------ Traceback (most recent call last): File "...testtools...runtest.py", line ..., in _run_user... - return fn(*args) + return fn(*args, **kwargs) File "...testtools...testcase.py", line ..., in _run_test_method return self._get_test_method()() File "...testtools...tests...test_testresult.py", line ..., in failed From 39fb562c5bc14bdfa302d1c2d3515e95524c347f Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Sun, 28 Nov 2010 12:25:39 +0000 Subject: [PATCH 13/49] Make cleanup running use run_user and make the run_user error handler understand MultipleExceptions. Makes the whole thing easier to follow. --- testtools/runtest.py | 38 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/testtools/runtest.py b/testtools/runtest.py index ae843a7..00e2f92 100644 --- a/testtools/runtest.py +++ b/testtools/runtest.py @@ -115,9 +115,7 @@ class RunTest(object): if self.exception_caught == self._run_user(self.case._run_setup, self.result): # Don't run the test method if we failed getting here. - e = self._run_cleanups(self.result) - if e is not None: - self._exceptions.append(e) + self._run_cleanups(self.result) return # Run everything from here on in. If any of the methods raise an # exception we'll have failed. @@ -133,9 +131,8 @@ class RunTest(object): failed = True finally: try: - e = self._run_user(self._run_cleanups, self.result) - if e is not None: - self._exceptions.append(e) + if self.exception_caught == self._run_user( + self._run_cleanups, self.result): failed = True finally: if not failed: @@ -162,26 +159,15 @@ class RunTest(object): :return: None if all cleanups ran without error, the most recently raised exception from the cleanups otherwise. """ - last_exception = None + failing = False while self.case._cleanups: function, arguments, keywordArguments = self.case._cleanups.pop() - try: - function(*arguments, **keywordArguments) - except KeyboardInterrupt: - raise - except: - exceptions = [sys.exc_info()] - while exceptions: - try: - exc_info = exceptions.pop() - if exc_info[0] is MultipleExceptions: - exceptions.extend(exc_info[1].args) - continue - self.case._report_traceback(exc_info) - last_exception = exc_info[1] - finally: - del exc_info - return last_exception + got_exception = self._run_user( + function, *arguments, **keywordArguments) + if got_exception == self.exception_caught: + failing = True + if failing: + return self.exception_caught def _got_user_exception(self, exc_info, tb_label='traceback'): """Called when user code raises an exception. @@ -190,6 +176,10 @@ class RunTest(object): :param tb_label: An optional string label for the error. If not specified, will default to 'traceback'. """ + if exc_info[0] is MultipleExceptions: + for sub_exc_info in exc_info[1].args: + self._got_user_exception(sub_exc_info, tb_label) + return self.exception_caught try: e = exc_info[1] self.case.onException(exc_info, tb_label=tb_label) From f1197a2f0515ff1f0d2e4bf95e0b2962ea79f8b8 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Sun, 28 Nov 2010 12:33:21 +0000 Subject: [PATCH 14/49] Docstring tweaks --- testtools/runtest.py | 54 +++++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/testtools/runtest.py b/testtools/runtest.py index 00e2f92..eb5801a 100644 --- a/testtools/runtest.py +++ b/testtools/runtest.py @@ -32,15 +32,15 @@ class RunTest(object): :ivar case: The test case that is to be run. :ivar result: The result object a case is reporting to. - :ivar handlers: A list of (ExceptionClass->handler code) for exceptions - that should be caught if raised from the user code. Exceptions that - are caught are checked against this list in first to last order. - There is a catchall of Exception at the end of the list, so to add - a new exception to the list, insert it at the front (which ensures that - it will be checked before any existing base classes in the list. If you - add multiple exceptions some of which are subclasses of each other, add - the most specific exceptions last (so they come before their parent - classes in the list). + :ivar handlers: A list of (ExceptionClass, handler_function) for + exceptions that should be caught if raised from the user + code. Exceptions that are caught are checked against this list in + first to last order. There is a catch-all of `Exception` at the end + of the list, so to add a new exception to the list, insert it at the + front (which ensures that it will be checked before any existing base + classes in the list. If you add multiple exceptions some of which are + subclasses of each other, add the most specific exceptions last (so + they come before their parent classes in the list). :ivar exception_caught: An object returned when _run_user catches an exception. :ivar _exceptions: A list of caught exceptions, used to do the single @@ -139,25 +139,13 @@ class RunTest(object): self.result.addSuccess(self.case, details=self.case.getDetails()) - def _run_user(self, fn, *args, **kwargs): - """Run a user supplied function. - - Exceptions are processed by self.handlers. - """ - try: - return fn(*args, **kwargs) - except KeyboardInterrupt: - raise - except: - return self._got_user_exception(sys.exc_info()) - def _run_cleanups(self, result): """Run the cleanups that have been added with addCleanup. See the docstring for addCleanup for more information. - :return: None if all cleanups ran without error, the most recently - raised exception from the cleanups otherwise. + :return: None if all cleanups ran without error, + `self.exception_caught` if there was an error. """ failing = False while self.case._cleanups: @@ -169,12 +157,32 @@ class RunTest(object): if failing: return self.exception_caught + def _run_user(self, fn, *args, **kwargs): + """Run a user supplied function. + + Exceptions are processed by `_got_user_exception`. + + :return: Either whatever 'fn' returns or `self.exception_caught` if + 'fn' raised an exception. + """ + try: + return fn(*args, **kwargs) + except KeyboardInterrupt: + raise + except: + return self._got_user_exception(sys.exc_info()) + def _got_user_exception(self, exc_info, tb_label='traceback'): """Called when user code raises an exception. + If 'exc_info' is a `MultipleExceptions`, then we recurse into it + unpacking the errors that it's made up from. + :param exc_info: A sys.exc_info() tuple for the user error. :param tb_label: An optional string label for the error. If not specified, will default to 'traceback'. + :return: `exception_caught` if we catch one of the exceptions that + have handlers in `self.handlers`, otherwise raise the error. """ if exc_info[0] is MultipleExceptions: for sub_exc_info in exc_info[1].args: From 61cdb7f6a17c62fec37a002c9e29061fafd109cb Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Sun, 28 Nov 2010 15:30:45 +0000 Subject: [PATCH 15/49] Test the current contract for wasSuccessful, make some of our component results enforce it. --- testtools/testresult/real.py | 10 ++++++ testtools/tests/test_testresult.py | 53 ++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/testtools/testresult/real.py b/testtools/testresult/real.py index 1493f90..44527c3 100644 --- a/testtools/testresult/real.py +++ b/testtools/testresult/real.py @@ -223,6 +223,13 @@ class MultiTestResult(TestResult): def done(self): return self._dispatch('done') + def wasSuccessful(self): + """Was this result successful? + + Only returns True if every constituent result was successful. + """ + return all(self._dispatch('wasSuccessful')) + class TextTestResult(TestResult): """A TestResult which outputs activity to a text stream.""" @@ -363,6 +370,9 @@ class ThreadsafeForwardingResult(TestResult): self._test_start = self._now() super(ThreadsafeForwardingResult, self).startTest(test) + def wasSuccessful(self): + return self.result.wasSuccessful() + class ExtendedToOriginalDecorator(object): """Permit new TestResult API code to degrade gracefully with old results. diff --git a/testtools/tests/test_testresult.py b/testtools/tests/test_testresult.py index 5686b7f..ebc70da 100644 --- a/testtools/tests/test_testresult.py +++ b/testtools/tests/test_testresult.py @@ -52,6 +52,11 @@ StringIO = try_imports(['StringIO.StringIO', 'io.StringIO']) class TestTestResultContract(TestCase): """Tests for the contract of TestResults.""" + def test_fresh_result_is_successful(self): + # A result is considered successful before any tests are run. + result = self.makeResult() + self.assertTrue(result.wasSuccessful()) + def test_addExpectedFailure(self): # Calling addExpectedFailure(test, exc_info) completes ok. result = self.makeResult() @@ -64,18 +69,42 @@ class TestTestResultContract(TestCase): result.startTest(self) result.addExpectedFailure(self, details={}) + def test_addExpectedFailure_is_success(self): + # addExpectedFailure does not fail the test run. + result = self.makeResult() + result.startTest(self) + result.addExpectedFailure(self, details={}) + result.stopTest(self) + self.assertTrue(result.wasSuccessful()) + def test_addError_details(self): # Calling addError(test, details=xxx) completes ok. result = self.makeResult() result.startTest(self) result.addError(self, details={}) + def test_addError_is_failure(self): + # addError fails the test run. + result = self.makeResult() + result.startTest(self) + result.addError(self, details={}) + result.stopTest(self) + self.assertFalse(result.wasSuccessful()) + def test_addFailure_details(self): # Calling addFailure(test, details=xxx) completes ok. result = self.makeResult() result.startTest(self) result.addFailure(self, details={}) + def test_addFailure_is_failure(self): + # addFailure fails the test run. + result = self.makeResult() + result.startTest(self) + result.addFailure(self, details={}) + result.stopTest(self) + self.assertFalse(result.wasSuccessful()) + def test_addSkipped(self): # Calling addSkip(test, reason) completes ok. result = self.makeResult() @@ -88,6 +117,14 @@ class TestTestResultContract(TestCase): result.startTest(self) result.addSkip(self, details={}) + def test_addSkip_is_success(self): + # addSkip does not fail the test run. + result = self.makeResult() + result.startTest(self) + result.addSkip(self, details={}) + result.stopTest(self) + self.assertTrue(result.wasSuccessful()) + def test_addUnexpectedSuccess(self): # Calling addUnexpectedSuccess(test) completes ok. result = self.makeResult() @@ -100,12 +137,28 @@ class TestTestResultContract(TestCase): result.startTest(self) result.addUnexpectedSuccess(self, details={}) + def test_addUnexpectedSuccess_is_success(self): + # addUnexpectedSuccess does not fail the test run. + result = self.makeResult() + result.startTest(self) + result.addUnexpectedSuccess(self, details={}) + result.stopTest(self) + self.assertTrue(result.wasSuccessful()) + def test_addSuccess_details(self): # Calling addSuccess(test) completes ok. result = self.makeResult() result.startTest(self) result.addSuccess(self, details={}) + def test_addSuccess_is_success(self): + # addSuccess does not fail the test run. + result = self.makeResult() + result.startTest(self) + result.addSuccess(self, details={}) + result.stopTest(self) + self.assertTrue(result.wasSuccessful()) + def test_startStopTestRun(self): # Calling startTestRun completes ok. result = self.makeResult() From 7f01d4b2ce73c2140d2fe281261e41b97cec7751 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Sun, 28 Nov 2010 15:35:27 +0000 Subject: [PATCH 16/49] Change the contract so that wasSuccessful should return False after unexpected successes. --- testtools/testresult/real.py | 12 ++++++++++++ testtools/tests/test_testresult.py | 6 +++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/testtools/testresult/real.py b/testtools/testresult/real.py index 44527c3..0324106 100644 --- a/testtools/testresult/real.py +++ b/testtools/testresult/real.py @@ -108,6 +108,18 @@ class TestResult(unittest.TestResult): """Called when a test was expected to fail, but succeed.""" self.unexpectedSuccesses.append(test) + def wasSuccessful(self): + """Has this result been successful so far? + + If there have been any errors, failures or unexpected successes, + return False. Otherwise, return True. + + Note: This differs from standard unittest in that we consider + unexpected successes to be equivalent to failures, rather than + successes. + """ + return not (self.errors or self.failures or self.unexpectedSuccesses) + if str_is_unicode: # Python 3 and IronPython strings are unicode, use parent class method _exc_info_to_unicode = unittest.TestResult._exc_info_to_string diff --git a/testtools/tests/test_testresult.py b/testtools/tests/test_testresult.py index ebc70da..5a9846d 100644 --- a/testtools/tests/test_testresult.py +++ b/testtools/tests/test_testresult.py @@ -137,13 +137,13 @@ class TestTestResultContract(TestCase): result.startTest(self) result.addUnexpectedSuccess(self, details={}) - def test_addUnexpectedSuccess_is_success(self): - # addUnexpectedSuccess does not fail the test run. + def test_addUnexpectedSuccess_is_failure(self): + # addUnexpectedSuccess fails the test run. result = self.makeResult() result.startTest(self) result.addUnexpectedSuccess(self, details={}) result.stopTest(self) - self.assertTrue(result.wasSuccessful()) + self.assertFalse(result.wasSuccessful()) def test_addSuccess_details(self): # Calling addSuccess(test) completes ok. From 216c7741b3119845d5f60f27b8b586913f0e2a4d Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Sun, 28 Nov 2010 15:45:08 +0000 Subject: [PATCH 17/49] Test the extended double's contract --- testtools/testresult/doubles.py | 12 +++++++++++- testtools/tests/test_testresult.py | 18 ++++++++++++------ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/testtools/testresult/doubles.py b/testtools/testresult/doubles.py index d231c91..bc6859c 100644 --- a/testtools/testresult/doubles.py +++ b/testtools/testresult/doubles.py @@ -1,4 +1,4 @@ -# Copyright (c) 2009 Jonathan M. Lange. See LICENSE for details. +# Copyright (c) 2009-2010 Jonathan M. Lange. See LICENSE for details. """Doubles of test result objects, useful for testing unittest code.""" @@ -15,6 +15,7 @@ class LoggingBase(object): def __init__(self): self._events = [] self.shouldStop = False + self._was_successful = True class Python26TestResult(LoggingBase): @@ -38,6 +39,9 @@ class Python26TestResult(LoggingBase): def stopTest(self, test): self._events.append(('stopTest', test)) + def wasSuccessful(self): + return self._was_successful + class Python27TestResult(Python26TestResult): """A precisely python 2.7 like test result, that logs.""" @@ -62,9 +66,11 @@ class ExtendedTestResult(Python27TestResult): """A test result like the proposed extended unittest result API.""" def addError(self, test, err=None, details=None): + self._was_successful = False self._events.append(('addError', test, err or details)) def addFailure(self, test, err=None, details=None): + self._was_successful = False self._events.append(('addFailure', test, err or details)) def addExpectedFailure(self, test, err=None, details=None): @@ -80,6 +86,7 @@ class ExtendedTestResult(Python27TestResult): self._events.append(('addSuccess', test)) def addUnexpectedSuccess(self, test, details=None): + self._was_successful = False if details is not None: self._events.append(('addUnexpectedSuccess', test, details)) else: @@ -93,3 +100,6 @@ class ExtendedTestResult(Python27TestResult): def time(self, time): self._events.append(('time', time)) + + def wasSuccessful(self): + return self._was_successful diff --git a/testtools/tests/test_testresult.py b/testtools/tests/test_testresult.py index 5a9846d..4ddd7de 100644 --- a/testtools/tests/test_testresult.py +++ b/testtools/tests/test_testresult.py @@ -73,7 +73,7 @@ class TestTestResultContract(TestCase): # addExpectedFailure does not fail the test run. result = self.makeResult() result.startTest(self) - result.addExpectedFailure(self, details={}) + result.addExpectedFailure(self, an_exc_info) result.stopTest(self) self.assertTrue(result.wasSuccessful()) @@ -101,7 +101,7 @@ class TestTestResultContract(TestCase): # addFailure fails the test run. result = self.makeResult() result.startTest(self) - result.addFailure(self, details={}) + result.addFailure(self, an_exc_info) result.stopTest(self) self.assertFalse(result.wasSuccessful()) @@ -121,7 +121,7 @@ class TestTestResultContract(TestCase): # addSkip does not fail the test run. result = self.makeResult() result.startTest(self) - result.addSkip(self, details={}) + result.addSkip(self, _u("Skipped for some reason")) result.stopTest(self) self.assertTrue(result.wasSuccessful()) @@ -137,11 +137,11 @@ class TestTestResultContract(TestCase): result.startTest(self) result.addUnexpectedSuccess(self, details={}) - def test_addUnexpectedSuccess_is_failure(self): + def test_addUnexpectedSuccess_was_successful(self): # addUnexpectedSuccess fails the test run. result = self.makeResult() result.startTest(self) - result.addUnexpectedSuccess(self, details={}) + result.addUnexpectedSuccess(self) result.stopTest(self) self.assertFalse(result.wasSuccessful()) @@ -155,7 +155,7 @@ class TestTestResultContract(TestCase): # addSuccess does not fail the test run. result = self.makeResult() result.startTest(self) - result.addSuccess(self, details={}) + result.addSuccess(self) result.stopTest(self) self.assertTrue(result.wasSuccessful()) @@ -192,6 +192,12 @@ class TestThreadSafeForwardingResultContract(TestTestResultContract): return ThreadsafeForwardingResult(target, result_semaphore) +class TestExtendedTestResultContract(TestTestResultContract): + + def makeResult(self): + return ExtendedTestResult() + + class TestTestResult(TestCase): """Tests for `TestResult`.""" From 9f81b9fa69ebab37a909d401e31a8c932267394b Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Sun, 28 Nov 2010 15:57:39 +0000 Subject: [PATCH 18/49] Thoroughly test the Python 2.6 and 2.7 contracts, make sure that the adapted versions of our implementations behave according to the extended contract --- testtools/testresult/doubles.py | 2 + testtools/testresult/real.py | 6 +- testtools/tests/test_testresult.py | 172 ++++++++++++++++++----------- 3 files changed, 112 insertions(+), 68 deletions(-) diff --git a/testtools/testresult/doubles.py b/testtools/testresult/doubles.py index bc6859c..20e19f5 100644 --- a/testtools/testresult/doubles.py +++ b/testtools/testresult/doubles.py @@ -22,9 +22,11 @@ class Python26TestResult(LoggingBase): """A precisely python 2.6 like test result, that logs.""" def addError(self, test, err): + self._was_successful = False self._events.append(('addError', test, err)) def addFailure(self, test, err): + self._was_successful = False self._events.append(('addFailure', test, err)) def addSuccess(self, test): diff --git a/testtools/testresult/real.py b/testtools/testresult/real.py index 0324106..4e2043d 100644 --- a/testtools/testresult/real.py +++ b/testtools/testresult/real.py @@ -398,11 +398,13 @@ class ExtendedToOriginalDecorator(object): def __init__(self, decorated): self.decorated = decorated + self._was_successful = True def __getattr__(self, name): return getattr(self.decorated, name) def addError(self, test, err=None, details=None): + self._was_successful = False self._check_args(err, details) if details is not None: try: @@ -427,6 +429,7 @@ class ExtendedToOriginalDecorator(object): return addExpectedFailure(test, err) def addFailure(self, test, err=None, details=None): + self._was_successful = False self._check_args(err, details) if details is not None: try: @@ -453,6 +456,7 @@ class ExtendedToOriginalDecorator(object): return addSkip(test, reason) def addUnexpectedSuccess(self, test, details=None): + self._was_successful = False outcome = getattr(self.decorated, 'addUnexpectedSuccess', None) if outcome is None: try: @@ -539,7 +543,7 @@ class ExtendedToOriginalDecorator(object): return method(a_datetime) def wasSuccessful(self): - return self.decorated.wasSuccessful() + return self._was_successful class _StringException(Exception): diff --git a/testtools/tests/test_testresult.py b/testtools/tests/test_testresult.py index 4ddd7de..577b931 100644 --- a/testtools/tests/test_testresult.py +++ b/testtools/tests/test_testresult.py @@ -49,54 +49,21 @@ from testtools.tests.helpers import ( StringIO = try_imports(['StringIO.StringIO', 'io.StringIO']) -class TestTestResultContract(TestCase): - """Tests for the contract of TestResults.""" +class Python26Contract(object): def test_fresh_result_is_successful(self): # A result is considered successful before any tests are run. result = self.makeResult() self.assertTrue(result.wasSuccessful()) - def test_addExpectedFailure(self): - # Calling addExpectedFailure(test, exc_info) completes ok. - result = self.makeResult() - result.startTest(self) - result.addExpectedFailure(self, an_exc_info) - - def test_addExpectedFailure_details(self): - # Calling addExpectedFailure(test, details=xxx) completes ok. - result = self.makeResult() - result.startTest(self) - result.addExpectedFailure(self, details={}) - - def test_addExpectedFailure_is_success(self): - # addExpectedFailure does not fail the test run. - result = self.makeResult() - result.startTest(self) - result.addExpectedFailure(self, an_exc_info) - result.stopTest(self) - self.assertTrue(result.wasSuccessful()) - - def test_addError_details(self): - # Calling addError(test, details=xxx) completes ok. - result = self.makeResult() - result.startTest(self) - result.addError(self, details={}) - def test_addError_is_failure(self): # addError fails the test run. result = self.makeResult() result.startTest(self) - result.addError(self, details={}) + result.addError(self, an_exc_info) result.stopTest(self) self.assertFalse(result.wasSuccessful()) - def test_addFailure_details(self): - # Calling addFailure(test, details=xxx) completes ok. - result = self.makeResult() - result.startTest(self) - result.addFailure(self, details={}) - def test_addFailure_is_failure(self): # addFailure fails the test run. result = self.makeResult() @@ -105,18 +72,37 @@ class TestTestResultContract(TestCase): result.stopTest(self) self.assertFalse(result.wasSuccessful()) + def test_addSuccess_is_success(self): + # addSuccess does not fail the test run. + result = self.makeResult() + result.startTest(self) + result.addSuccess(self) + result.stopTest(self) + self.assertTrue(result.wasSuccessful()) + + +class Python27Contract(Python26Contract): + + def test_addExpectedFailure(self): + # Calling addExpectedFailure(test, exc_info) completes ok. + result = self.makeResult() + result.startTest(self) + result.addExpectedFailure(self, an_exc_info) + + def test_addExpectedFailure_is_success(self): + # addExpectedFailure does not fail the test run. + result = self.makeResult() + result.startTest(self) + result.addExpectedFailure(self, an_exc_info) + result.stopTest(self) + self.assertTrue(result.wasSuccessful()) + def test_addSkipped(self): # Calling addSkip(test, reason) completes ok. result = self.makeResult() result.startTest(self) result.addSkip(self, _u("Skipped for some reason")) - def test_addSkipped_details(self): - # Calling addSkip(test, reason) completes ok. - result = self.makeResult() - result.startTest(self) - result.addSkip(self, details={}) - def test_addSkip_is_success(self): # addSkip does not fail the test run. result = self.makeResult() @@ -131,32 +117,12 @@ class TestTestResultContract(TestCase): result.startTest(self) result.addUnexpectedSuccess(self) - def test_addUnexpectedSuccess_details(self): - # Calling addUnexpectedSuccess(test) completes ok. - result = self.makeResult() - result.startTest(self) - result.addUnexpectedSuccess(self, details={}) - def test_addUnexpectedSuccess_was_successful(self): - # addUnexpectedSuccess fails the test run. + # addUnexpectedSuccess does not the test run in Python 2.7. result = self.makeResult() result.startTest(self) result.addUnexpectedSuccess(self) result.stopTest(self) - self.assertFalse(result.wasSuccessful()) - - def test_addSuccess_details(self): - # Calling addSuccess(test) completes ok. - result = self.makeResult() - result.startTest(self) - result.addSuccess(self, details={}) - - def test_addSuccess_is_success(self): - # addSuccess does not fail the test run. - result = self.makeResult() - result.startTest(self) - result.addSuccess(self) - result.stopTest(self) self.assertTrue(result.wasSuccessful()) def test_startStopTestRun(self): @@ -166,25 +132,73 @@ class TestTestResultContract(TestCase): result.stopTestRun() -class TestTestResultContract(TestTestResultContract): +class TestResultContract(Python27Contract): + """Tests for the contract of TestResults.""" + + def test_addExpectedFailure_details(self): + # Calling addExpectedFailure(test, details=xxx) completes ok. + result = self.makeResult() + result.startTest(self) + result.addExpectedFailure(self, details={}) + + def test_addError_details(self): + # Calling addError(test, details=xxx) completes ok. + result = self.makeResult() + result.startTest(self) + result.addError(self, details={}) + + def test_addFailure_details(self): + # Calling addFailure(test, details=xxx) completes ok. + result = self.makeResult() + result.startTest(self) + result.addFailure(self, details={}) + + def test_addSkipped_details(self): + # Calling addSkip(test, reason) completes ok. + result = self.makeResult() + result.startTest(self) + result.addSkip(self, details={}) + + def test_addUnexpectedSuccess_details(self): + # Calling addUnexpectedSuccess(test) completes ok. + result = self.makeResult() + result.startTest(self) + result.addUnexpectedSuccess(self, details={}) + + def test_addSuccess_details(self): + # Calling addSuccess(test) completes ok. + result = self.makeResult() + result.startTest(self) + result.addSuccess(self, details={}) + + def test_addUnexpectedSuccess_was_successful(self): + # addUnexpectedSuccess fails test run in testtools. + result = self.makeResult() + result.startTest(self) + result.addUnexpectedSuccess(self) + result.stopTest(self) + self.assertFalse(result.wasSuccessful()) + + +class TestTestResultContract(TestCase, TestResultContract): def makeResult(self): return TestResult() -class TestMultiTestresultContract(TestTestResultContract): +class TestMultiTestresultContract(TestCase, TestResultContract): def makeResult(self): return MultiTestResult(TestResult(), TestResult()) -class TestTextTestResultContract(TestTestResultContract): +class TestTextTestResultContract(TestCase, TestResultContract): def makeResult(self): return TextTestResult(StringIO()) -class TestThreadSafeForwardingResultContract(TestTestResultContract): +class TestThreadSafeForwardingResultContract(TestCase, TestResultContract): def makeResult(self): result_semaphore = threading.Semaphore(1) @@ -192,12 +206,36 @@ class TestThreadSafeForwardingResultContract(TestTestResultContract): return ThreadsafeForwardingResult(target, result_semaphore) -class TestExtendedTestResultContract(TestTestResultContract): +class TestExtendedTestResultContract(TestCase, TestResultContract): def makeResult(self): return ExtendedTestResult() +class TestPython26TestResultContract(TestCase, Python26Contract): + + def makeResult(self): + return Python26TestResult() + + +class TestAdaptedPython26TestResultContract(TestCase, TestResultContract): + + def makeResult(self): + return ExtendedToOriginalDecorator(Python26TestResult()) + + +class TestPython27TestResultContract(TestCase, Python27Contract): + + def makeResult(self): + return Python27TestResult() + + +class TestAdaptedPython27TestResultContract(TestCase, TestResultContract): + + def makeResult(self): + return ExtendedToOriginalDecorator(Python27TestResult()) + + class TestTestResult(TestCase): """Tests for `TestResult`.""" From 645c2743c5556be50251bb28b6cd9e2289ff4dba Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Sun, 28 Nov 2010 16:04:41 +0000 Subject: [PATCH 19/49] Update NEWS --- NEWS | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index bbd725e..dfefe98 100644 --- a/NEWS +++ b/NEWS @@ -8,7 +8,9 @@ Changes ------- * addUnexpectedSuccess is translated to addFailure for test results that don't - know about addUnexpectedSuccess. (Jonathan Lange, #654474) + know about addUnexpectedSuccess. Further, it fails the entire result for + all testtools TestResults (i.e. wasSuccessful() returns False after + addUnexpectedSuccess has been called). (Jonathan Lange, #654474) Improvements ------------ From cd4bc797e2b067bb741782bd83b30c94ceebcd2d Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Sun, 28 Nov 2010 23:55:14 +0000 Subject: [PATCH 20/49] minor nits --- testtools/tests/test_testresult.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testtools/tests/test_testresult.py b/testtools/tests/test_testresult.py index 577b931..d812f5f 100644 --- a/testtools/tests/test_testresult.py +++ b/testtools/tests/test_testresult.py @@ -118,7 +118,7 @@ class Python27Contract(Python26Contract): result.addUnexpectedSuccess(self) def test_addUnexpectedSuccess_was_successful(self): - # addUnexpectedSuccess does not the test run in Python 2.7. + # addUnexpectedSuccess does not fail the test run in Python 2.7. result = self.makeResult() result.startTest(self) result.addUnexpectedSuccess(self) @@ -186,7 +186,7 @@ class TestTestResultContract(TestCase, TestResultContract): return TestResult() -class TestMultiTestresultContract(TestCase, TestResultContract): +class TestMultiTestResultContract(TestCase, TestResultContract): def makeResult(self): return MultiTestResult(TestResult(), TestResult()) From aca08db792b1d8319f9081c927e2b0632c302d98 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Mon, 29 Nov 2010 00:02:44 +0000 Subject: [PATCH 21/49] Make MultiTestResult at all usable in Python 3 --- testtools/testresult/real.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testtools/testresult/real.py b/testtools/testresult/real.py index 4e2043d..1da0f54 100644 --- a/testtools/testresult/real.py +++ b/testtools/testresult/real.py @@ -194,7 +194,7 @@ class MultiTestResult(TestResult): def __init__(self, *results): TestResult.__init__(self) - self._results = map(ExtendedToOriginalDecorator, results) + self._results = list(map(ExtendedToOriginalDecorator, results)) def _dispatch(self, message, *args, **kwargs): return tuple( From 57d23dc6f4a8e30b73f5eeb41f3416d812801b9c Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Mon, 29 Nov 2010 00:08:54 +0000 Subject: [PATCH 22/49] startTestRun resets unexpected successes and other errors --- testtools/testresult/doubles.py | 4 ++++ testtools/testresult/real.py | 4 ++++ testtools/tests/test_testresult.py | 23 +++++++++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/testtools/testresult/doubles.py b/testtools/testresult/doubles.py index 20e19f5..7e4a2c9 100644 --- a/testtools/testresult/doubles.py +++ b/testtools/testresult/doubles.py @@ -97,6 +97,10 @@ class ExtendedTestResult(Python27TestResult): def progress(self, offset, whence): self._events.append(('progress', offset, whence)) + def startTestRun(self): + super(ExtendedTestResult, self).startTestRun() + self._was_successful = True + def tags(self, new_tags, gone_tags): self._events.append(('tags', new_tags, gone_tags)) diff --git a/testtools/testresult/real.py b/testtools/testresult/real.py index 1da0f54..fd712a2 100644 --- a/testtools/testresult/real.py +++ b/testtools/testresult/real.py @@ -160,6 +160,9 @@ class TestResult(unittest.TestResult): New in python 2.7 """ + self.unexpectedSuccesses = [] + self.errors = [] + self.failures = [] def stopTestRun(self): """Called after a test run completes @@ -513,6 +516,7 @@ class ExtendedToOriginalDecorator(object): return self.decorated.startTest(test) def startTestRun(self): + self._was_successful = True try: return self.decorated.startTestRun() except AttributeError: diff --git a/testtools/tests/test_testresult.py b/testtools/tests/test_testresult.py index d812f5f..e45b0d2 100644 --- a/testtools/tests/test_testresult.py +++ b/testtools/tests/test_testresult.py @@ -179,6 +179,29 @@ class TestResultContract(Python27Contract): result.stopTest(self) self.assertFalse(result.wasSuccessful()) + def test_startTestRun_resets_unexpected_success(self): + result = self.makeResult() + result.startTest(self) + result.addUnexpectedSuccess(self) + result.stopTest(self) + result.startTestRun() + self.assertTrue(result.wasSuccessful()) + + def test_startTestRun_resets_failure(self): + result = self.makeResult() + result.startTest(self) + result.addFailure(self, an_exc_info) + result.stopTest(self) + result.startTestRun() + self.assertTrue(result.wasSuccessful()) + + def test_startTestRun_resets_errors(self): + result = self.makeResult() + result.startTest(self) + result.addError(self, an_exc_info) + result.stopTest(self) + result.startTestRun() + self.assertTrue(result.wasSuccessful()) class TestTestResultContract(TestCase, TestResultContract): From 56a754e0a50c1a5239d3c7bcdb2fb116f723de05 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Mon, 29 Nov 2010 00:09:59 +0000 Subject: [PATCH 23/49] NEWS update --- NEWS | 3 +++ 1 file changed, 3 insertions(+) diff --git a/NEWS b/NEWS index dfefe98..5c24a94 100644 --- a/NEWS +++ b/NEWS @@ -12,6 +12,9 @@ Changes all testtools TestResults (i.e. wasSuccessful() returns False after addUnexpectedSuccess has been called). (Jonathan Lange, #654474) +* startTestRun will reset any errors on the result. That is, wasSuccessful() + will always return True immediately after startTestRun() is called. + Improvements ------------ From ce68114a817ca42b457bc8a49f1bed02af6b2cd2 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Mon, 29 Nov 2010 00:20:27 +0000 Subject: [PATCH 24/49] Actually report the error in TextTestResult (thanks mgz) --- testtools/testresult/real.py | 7 ++++++- testtools/tests/test_testresult.py | 20 +++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/testtools/testresult/real.py b/testtools/testresult/real.py index fd712a2..793a9ef 100644 --- a/testtools/testresult/real.py +++ b/testtools/testresult/real.py @@ -280,6 +280,10 @@ class TextTestResult(TestResult): stop = self._now() self._show_list('ERROR', self.errors) self._show_list('FAIL', self.failures) + for test in self.unexpectedSuccesses: + self.stream.write( + "%sUNEXPECTED SUCCESS: %s\n%s" % ( + self.sep1, test.id(), self.sep2)) self.stream.write("Ran %d test%s in %.3fs\n\n" % (self.testsRun, plural, self._delta_to_float(stop - self.__start))) @@ -289,7 +293,8 @@ class TextTestResult(TestResult): self.stream.write("FAILED (") details = [] details.append("failures=%d" % ( - len(self.failures) + len(self.errors))) + len(self.failures) + len(self.errors) + + len(self.unexpectedSuccesses))) self.stream.write(", ".join(details)) self.stream.write(")\n") super(TextTestResult, self).stopTestRun() diff --git a/testtools/tests/test_testresult.py b/testtools/tests/test_testresult.py index e45b0d2..aefb42d 100644 --- a/testtools/tests/test_testresult.py +++ b/testtools/tests/test_testresult.py @@ -428,6 +428,12 @@ class TestTextTestResult(TestCase): self.fail("yo!") return Test("failed") + def make_unexpectedly_successful_test(self): + class Test(TestCase): + def succeeded(self): + self.expectFailure("yo!", lambda: None) + return Test("succeeded") + def make_test(self): class Test(TestCase): def test(self): @@ -513,9 +519,18 @@ class TestTextTestResult(TestCase): self.assertThat(self.getvalue(), DocTestMatches("...\n\nFAILED (failures=1)\n", doctest.ELLIPSIS)) + def test_stopTestRun_not_successful_unexpected_success(self): + test = self.make_unexpectedly_successful_test() + self.result.startTestRun() + test.run(self.result) + self.result.stopTestRun() + self.assertThat(self.getvalue(), + DocTestMatches("...\n\nFAILED (failures=1)\n", doctest.ELLIPSIS)) + def test_stopTestRun_shows_details(self): self.result.startTestRun() self.make_erroring_test().run(self.result) + self.make_unexpectedly_successful_test().run(self.result) self.make_failing_test().run(self.result) self.reset_output() self.result.stopTestRun() @@ -548,7 +563,10 @@ Traceback (most recent call last): self.fail("yo!") AssertionError: yo! ------------ -...""", doctest.ELLIPSIS)) +====================================================================== +UNEXPECTED SUCCESS: testtools.tests.test_testresult.Test.succeeded +---------------------------------------------------------------------- +...""", doctest.ELLIPSIS | doctest.REPORT_NDIFF)) class TestThreadSafeForwardingResult(TestWithFakeExceptions): From e4320c3f38feac669ad4780f405ce5092d94a3a0 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Mon, 29 Nov 2010 00:21:59 +0000 Subject: [PATCH 25/49] Fix a inaccurate unexpected success, hidden by our broken Python 3 support --- testtools/tests/test_compat.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/testtools/tests/test_compat.py b/testtools/tests/test_compat.py index 440e24c..8569538 100644 --- a/testtools/tests/test_compat.py +++ b/testtools/tests/test_compat.py @@ -19,6 +19,7 @@ from testtools.compat import ( ) from testtools.matchers import ( MatchesException, + Not, Raises, ) @@ -196,34 +197,34 @@ class TestUnicodeOutputStream(testtools.TestCase): super(TestUnicodeOutputStream, self).setUp() if sys.platform == "cli": self.skip("IronPython shouldn't wrap streams to do encoding") - + def test_no_encoding_becomes_ascii(self): """A stream with no encoding attribute gets ascii/replace strings""" sout = _FakeOutputStream() unicode_output_stream(sout).write(self.uni) self.assertEqual([_b("pa???n")], sout.writelog) - + def test_encoding_as_none_becomes_ascii(self): """A stream with encoding value of None gets ascii/replace strings""" sout = _FakeOutputStream() sout.encoding = None unicode_output_stream(sout).write(self.uni) self.assertEqual([_b("pa???n")], sout.writelog) - + def test_bogus_encoding_becomes_ascii(self): """A stream with a bogus encoding gets ascii/replace strings""" sout = _FakeOutputStream() sout.encoding = "bogus" unicode_output_stream(sout).write(self.uni) self.assertEqual([_b("pa???n")], sout.writelog) - + def test_partial_encoding_replace(self): """A string which can be partly encoded correctly should be""" sout = _FakeOutputStream() sout.encoding = "iso-8859-7" unicode_output_stream(sout).write(self.uni) self.assertEqual([_b("pa?\xe8?n")], sout.writelog) - + def test_unicode_encodings_not_wrapped(self): """A unicode encoding is left unwrapped as needs no error handler""" sout = _FakeOutputStream() @@ -232,7 +233,7 @@ class TestUnicodeOutputStream(testtools.TestCase): sout = _FakeOutputStream() sout.encoding = "utf-16-be" self.assertIs(unicode_output_stream(sout), sout) - + def test_stringio(self): """A StringIO object should maybe get an ascii native str type""" try: @@ -246,7 +247,7 @@ class TestUnicodeOutputStream(testtools.TestCase): if newio: self.expectFailure("Python 3 StringIO expects text not bytes", self.assertThat, lambda: soutwrapper.write(self.uni), - Raises(MatchesException(TypeError))) + Not(Raises(MatchesException(TypeError)))) soutwrapper.write(self.uni) self.assertEqual("pa???n", sout.getvalue()) From 748fefdc122ac40648b31113c85846ee0851a7fd Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 30 Nov 2010 09:32:00 +1300 Subject: [PATCH 26/49] * ``testtools.run`` now supports ``-l`` to list tests rather than executing them. This is useful for integration with external test analysis/processing tools like subunit and testrepository. (Robert Collins) --- NEWS | 13 ++++++++---- testtools/run.py | 42 ++++++++++++++++++++++++++----------- testtools/tests/__init__.py | 7 +++---- testtools/tests/test_run.py | 39 ++++++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 20 deletions(-) create mode 100644 testtools/tests/test_run.py diff --git a/NEWS b/NEWS index 50889c6..4b62409 100644 --- a/NEWS +++ b/NEWS @@ -42,6 +42,11 @@ Improvements * ``MatchesException`` added to the ``testtools.matchers`` module - matches an exception class and parameters. (Robert Collins) +* New ``KeysEqual`` matcher. (Jonathan Lange) + +* New helpers for conditionally importing modules, ``try_import`` and + ``try_imports``. (Jonathan Lange) + * ``Raises`` added to the ``testtools.matchers`` module - matches if the supplied callable raises, and delegates to an optional matcher for validation of the exception. (Robert Collins) @@ -53,16 +58,16 @@ Improvements * ``testools.TestCase.useFixture`` has been added to glue with fixtures nicely. (Robert Collins) +* ``testtools.run`` now supports ``-l`` to list tests rather than executing + them. This is useful for integration with external test analysis/processing + tools like subunit and testrepository. (Robert Collins) + * Update documentation to say how to use testtools.run() on Python 2.4. (Jonathan Lange, #501174) * ``text_content`` conveniently converts a Python string to a Content object. (Jonathan Lange, James Westby) -* New ``KeysEqual`` matcher. (Jonathan Lange) - -* New helpers for conditionally importing modules, ``try_import`` and - ``try_imports``. (Jonathan Lange) 0.9.7 diff --git a/testtools/run.py b/testtools/run.py index 091c3c7..b99510f 100755 --- a/testtools/run.py +++ b/testtools/run.py @@ -14,6 +14,7 @@ import sys from testtools import TextTestResult from testtools.compat import classtypes, istext, unicode_output_stream +from testtools.testsuite import iterate_tests defaultTestLoader = unittest.defaultTestLoader @@ -34,9 +35,12 @@ else: class TestToolsTestRunner(object): """ A thunk object to support unittest.TestProgram.""" + def __init__(self, stdout): + self.stdout = stdout + def run(self, test): "Run the given test case or test suite." - result = TextTestResult(unicode_output_stream(sys.stdout)) + result = TextTestResult(unicode_output_stream(self.stdout)) result.startTestRun() try: return test.run(result) @@ -70,6 +74,7 @@ Options: -h, --help Show this message -v, --verbose Verbose output -q, --quiet Minimal output + -l, --list List tests rather than executing them. %(failfast)s%(catchbreak)s%(buffer)s Examples: %(progName)s test_module - run tests from test_module @@ -87,6 +92,7 @@ Options: -p pattern Pattern to match test files ('test*.py' default) -t directory Top level directory of project (default to start directory) + -l, --list List tests rather than executing them. For test discovery all test modules must be importable from the top level directory of the project. @@ -102,11 +108,13 @@ class TestProgram(object): # defaults for testing failfast = catchbreak = buffer = progName = None - def __init__(self, module='__main__', defaultTest=None, argv=None, + def __init__(self, module=__name__, defaultTest=None, argv=None, testRunner=None, testLoader=defaultTestLoader, exit=True, verbosity=1, failfast=None, catchbreak=None, - buffer=None): - if istext(module): + buffer=None, stdout=None): + if module == __name__: + self.module = None + elif istext(module): self.module = __import__(module) for part in module.split('.')[1:]: self.module = getattr(self.module, part) @@ -121,6 +129,7 @@ class TestProgram(object): self.verbosity = verbosity self.buffer = buffer self.defaultTest = defaultTest + self.listtests = False self.testRunner = testRunner self.testLoader = testLoader progName = argv[0] @@ -131,7 +140,11 @@ class TestProgram(object): progName = os.path.basename(argv[0]) self.progName = progName self.parseArgs(argv) - self.runTests() + if not self.listtests: + self.runTests() + else: + for test in iterate_tests(self.test): + stdout.write('%s\n' % test.id()) def usageExit(self, msg=None): if msg: @@ -153,9 +166,10 @@ class TestProgram(object): return import getopt - long_opts = ['help', 'verbose', 'quiet', 'failfast', 'catch', 'buffer'] + long_opts = ['help', 'verbose', 'quiet', 'failfast', 'catch', 'buffer', + 'list'] try: - options, args = getopt.getopt(argv[1:], 'hHvqfcb', long_opts) + options, args = getopt.getopt(argv[1:], 'hHvqfcbl', long_opts) for opt, value in options: if opt in ('-h','-H','--help'): self.usageExit() @@ -175,14 +189,13 @@ class TestProgram(object): if self.buffer is None: self.buffer = True # Should this raise an exception if -b is not valid? + if opt in ('-l', '--list'): + self.listtests = True if len(args) == 0 and self.defaultTest is None: # createTests will load tests from self.module self.testNames = None elif len(args) > 0: self.testNames = args - if __name__ == '__main__': - # to support python -m unittest ... - self.module = None else: self.testNames = (self.defaultTest,) self.createTests() @@ -225,6 +238,8 @@ class TestProgram(object): help="Pattern to match tests ('test*.py' default)") parser.add_option('-t', '--top-level-directory', dest='top', default=None, help='Top level directory of project (defaults to start directory)') + parser.add_option('-l', '--list', dest='listtests', default=False, + help='List tests rather than running them.') options, args = parser.parse_args(argv) if len(args) > 3: @@ -241,6 +256,7 @@ class TestProgram(object): self.catchbreak = options.catchbreak if self.buffer is None: self.buffer = options.buffer + self.listtests = options.listtests if options.verbose: self.verbosity = 2 @@ -274,7 +290,9 @@ class TestProgram(object): sys.exit(not self.result.wasSuccessful()) ################ +def main(argv, stdout): + runner = TestToolsTestRunner(stdout) + program = TestProgram(argv=argv, testRunner=runner, stdout=stdout) if __name__ == '__main__': - runner = TestToolsTestRunner() - program = TestProgram(argv=sys.argv, testRunner=runner) + main(sys.argv, sys.stdout) diff --git a/testtools/tests/__init__.py b/testtools/tests/__init__.py index b3b9b4e..ac3c218 100644 --- a/testtools/tests/__init__.py +++ b/testtools/tests/__init__.py @@ -15,13 +15,13 @@ def test_suite(): test_helpers, test_matchers, test_monkey, + test_run, test_runtest, test_spinner, test_testtools, test_testresult, test_testsuite, ) - suites = [] modules = [ test_compat, test_content, @@ -31,12 +31,11 @@ def test_suite(): test_helpers, test_matchers, test_monkey, - test_runtest, + test_run, test_spinner, test_testresult, test_testsuite, test_testtools, ] - for module in modules: - suites.append(getattr(module, 'test_suite')()) + suites = map(lambda x:x.test_suite(), modules) return unittest.TestSuite(suites) diff --git a/testtools/tests/test_run.py b/testtools/tests/test_run.py new file mode 100644 index 0000000..8167f4e --- /dev/null +++ b/testtools/tests/test_run.py @@ -0,0 +1,39 @@ +# Copyright (c) 2010 Testtools authors. See LICENSE for details. + +"""Tests for the test runner logic.""" + +import StringIO + +from testtools.helpers import try_import +fixtures = try_import('fixtures') + +import testtools +from testtools import TestCase, run + + +class TestRun(TestCase): + + def test_run_list(self): + if fixtures is None: + self.skipTest("Need fixtures") + package = self.useFixture(fixtures.PythonPackage( + 'runexample', [('__init__.py', """ +from testtools import TestCase + +class TestFoo(TestCase): + def test_bar(self): + pass + def test_quux(self): + pass +def test_suite(): + from unittest import TestLoader + return TestLoader().loadTestsFromName(__name__) +""")])) + testtools.__path__.append(package.base) + self.addCleanup(testtools.__path__.remove, package.base) + out = StringIO.StringIO() + run.main(['-l', 'testtools.runexample.test_suite'], out) + +def test_suite(): + from unittest import TestLoader + return TestLoader().loadTestsFromName(__name__) From 8bb716ac9d9bada4e71f86d3a99c2f89ff5171e1 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 30 Nov 2010 10:18:12 +1300 Subject: [PATCH 27/49] Forgot to actually assert that list worked. --- testtools/tests/test_run.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testtools/tests/test_run.py b/testtools/tests/test_run.py index 8167f4e..b680a91 100644 --- a/testtools/tests/test_run.py +++ b/testtools/tests/test_run.py @@ -33,6 +33,9 @@ def test_suite(): self.addCleanup(testtools.__path__.remove, package.base) out = StringIO.StringIO() run.main(['-l', 'testtools.runexample.test_suite'], out) + self.assertEqual("""testtools.runexample.TestFoo.test_bar +testtools.runexample.TestFoo.test_quux +""", out.getvalue()) def test_suite(): from unittest import TestLoader From e2e918499cb4aa70fb21b27ba28f8a8635e78fdc Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 30 Nov 2010 11:45:14 +1300 Subject: [PATCH 28/49] Refactor for reusability. --- testtools/tests/test_run.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/testtools/tests/test_run.py b/testtools/tests/test_run.py index b680a91..22f3dce 100644 --- a/testtools/tests/test_run.py +++ b/testtools/tests/test_run.py @@ -11,12 +11,12 @@ import testtools from testtools import TestCase, run -class TestRun(TestCase): +if fixtures: + class SampleTestFixture(fixtures.Fixture): + """Creates testtools.runexample temporarily.""" - def test_run_list(self): - if fixtures is None: - self.skipTest("Need fixtures") - package = self.useFixture(fixtures.PythonPackage( + def __init__(self): + self.package = fixtures.PythonPackage( 'runexample', [('__init__.py', """ from testtools import TestCase @@ -28,9 +28,21 @@ class TestFoo(TestCase): def test_suite(): from unittest import TestLoader return TestLoader().loadTestsFromName(__name__) -""")])) - testtools.__path__.append(package.base) - self.addCleanup(testtools.__path__.remove, package.base) +""")]) + + def setUp(self): + super(SampleTestFixture, self).setUp() + self.useFixture(self.package) + testtools.__path__.append(self.package.base) + self.addCleanup(testtools.__path__.remove, self.package.base) + + +class TestRun(TestCase): + + def test_run_list(self): + if fixtures is None: + self.skipTest("Need fixtures") + package = self.useFixture(SampleTestFixture()) out = StringIO.StringIO() run.main(['-l', 'testtools.runexample.test_suite'], out) self.assertEqual("""testtools.runexample.TestFoo.test_bar From 826d0624145beaf2fffa4892f75731b2f66f6c27 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 30 Nov 2010 12:09:22 +1300 Subject: [PATCH 29/49] Fix thinko - first param is program is ignored. --- testtools/tests/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testtools/tests/test_run.py b/testtools/tests/test_run.py index 22f3dce..73cddb0 100644 --- a/testtools/tests/test_run.py +++ b/testtools/tests/test_run.py @@ -44,7 +44,7 @@ class TestRun(TestCase): self.skipTest("Need fixtures") package = self.useFixture(SampleTestFixture()) out = StringIO.StringIO() - run.main(['-l', 'testtools.runexample.test_suite'], out) + run.main(['prog', '-l', 'testtools.runexample.test_suite'], out) self.assertEqual("""testtools.runexample.TestFoo.test_bar testtools.runexample.TestFoo.test_quux """, out.getvalue()) From 3353c78b0fa0a4280963f1b9faf9d81db7e5d2f0 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 30 Nov 2010 12:46:10 +1300 Subject: [PATCH 30/49] * ``testtools.run`` now supports ``--load-list``, which takes a file containing test ids, one per line, and intersects those ids with the tests found. This allows fine grained control of what tests are run even when the tests cannot be named as objects to import (e.g. due to test parameterisation via testscenarios). (Robert Collins) --- NEWS | 6 ++++++ testtools/run.py | 38 ++++++++++++++++++++++++++++++++++--- testtools/tests/test_run.py | 23 ++++++++++++++++++++++ 3 files changed, 64 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 4b62409..dcabfbd 100644 --- a/NEWS +++ b/NEWS @@ -62,6 +62,12 @@ Improvements them. This is useful for integration with external test analysis/processing tools like subunit and testrepository. (Robert Collins) +* ``testtools.run`` now supports ``--load-list``, which takes a file containing + test ids, one per line, and intersects those ids with the tests found. This + allows fine grained control of what tests are run even when the tests cannot + be named as objects to import (e.g. due to test parameterisation via + testscenarios). (Robert Collins) + * Update documentation to say how to use testtools.run() on Python 2.4. (Jonathan Lange, #501174) diff --git a/testtools/run.py b/testtools/run.py index b99510f..da4496a 100755 --- a/testtools/run.py +++ b/testtools/run.py @@ -62,6 +62,12 @@ class TestToolsTestRunner(object): # removed. # - A tweak has been added to detect 'python -m *.run' and use a # better progName in that case. +# - self.module is more comprehensively set to None when being invoked from +# the commandline - __name__ is used as a sentinel value. +# - --list has been added which can list tests (should be upstreamed). +# - --load-list has been added which can reduce the tests used (should be +# upstreamed). +# - The limitation of using getopt is declared to the user. FAILFAST = " -f, --failfast Stop on first failure\n" CATCHBREAK = " -c, --catch Catch control-C and display results\n" @@ -75,14 +81,16 @@ Options: -v, --verbose Verbose output -q, --quiet Minimal output -l, --list List tests rather than executing them. + --load-list Specifies a file containing test ids, only tests matching + those ids are executed. %(failfast)s%(catchbreak)s%(buffer)s Examples: %(progName)s test_module - run tests from test_module %(progName)s module.TestClass - run tests from module.TestClass %(progName)s module.Class.test_method - run specified test method -[tests] can be a list of any number of test modules, classes and test -methods. +All options must come before [tests]. [tests] can be a list of any number of +test modules, classes and test methods. Alternative Usage: %(progName)s discover [options] @@ -93,6 +101,8 @@ Options: -t directory Top level directory of project (default to start directory) -l, --list List tests rather than executing them. + --load-list Specifies a file containing test ids, only tests matching + those ids are executed. For test discovery all test modules must be importable from the top level directory of the project. @@ -130,6 +140,7 @@ class TestProgram(object): self.buffer = buffer self.defaultTest = defaultTest self.listtests = False + self.load_list = None self.testRunner = testRunner self.testLoader = testLoader progName = argv[0] @@ -140,6 +151,22 @@ class TestProgram(object): progName = os.path.basename(argv[0]) self.progName = progName self.parseArgs(argv) + if self.load_list: + # TODO: preserve existing suites (like testresources does in + # OptimisingTestSuite.add, but with a standard protocol). + # This is needed because the load_tests hook allows arbitrary + # suites, even if that is rarely used. + source = file(self.load_list, 'rb') + try: + lines = source.readlines() + finally: + source.close() + test_ids = set(line.strip() for line in lines) + filtered = unittest.TestSuite() + for test in iterate_tests(self.test): + if test.id() in test_ids: + filtered.addTest(test) + self.test = filtered if not self.listtests: self.runTests() else: @@ -167,7 +194,7 @@ class TestProgram(object): import getopt long_opts = ['help', 'verbose', 'quiet', 'failfast', 'catch', 'buffer', - 'list'] + 'list', 'load-list='] try: options, args = getopt.getopt(argv[1:], 'hHvqfcbl', long_opts) for opt, value in options: @@ -191,6 +218,8 @@ class TestProgram(object): # Should this raise an exception if -b is not valid? if opt in ('-l', '--list'): self.listtests = True + if opt == '--load-list': + self.load_list = value if len(args) == 0 and self.defaultTest is None: # createTests will load tests from self.module self.testNames = None @@ -240,6 +269,8 @@ class TestProgram(object): help='Top level directory of project (defaults to start directory)') parser.add_option('-l', '--list', dest='listtests', default=False, help='List tests rather than running them.') + parser.add_option('--load-list', dest='load_list', default=None, + help='Specify a filename containing the test ids to use.') options, args = parser.parse_args(argv) if len(args) > 3: @@ -257,6 +288,7 @@ class TestProgram(object): if self.buffer is None: self.buffer = options.buffer self.listtests = options.listtests + self.load_list = options.load_list if options.verbose: self.verbosity = 2 diff --git a/testtools/tests/test_run.py b/testtools/tests/test_run.py index 73cddb0..5087527 100644 --- a/testtools/tests/test_run.py +++ b/testtools/tests/test_run.py @@ -49,6 +49,29 @@ class TestRun(TestCase): testtools.runexample.TestFoo.test_quux """, out.getvalue()) + def test_run_load_list(self): + if fixtures is None: + self.skipTest("Need fixtures") + package = self.useFixture(SampleTestFixture()) + out = StringIO.StringIO() + # We load two tests - one that exists and one that doesn't, and we + # should get the one that exists and neither the one that doesn't nor + # the unmentioned one that does. + tempdir = self.useFixture(fixtures.TempDir()) + tempname = tempdir.path + '/tests.list' + f = open(tempname, 'wb') + try: + f.write(""" +testtools.runexample.TestFoo.test_bar +testtools.runexample.missingtest +""") + finally: + f.close() + run.main(['prog', '-l', '--load-list', tempname, + 'testtools.runexample.test_suite'], out) + self.assertEqual("""testtools.runexample.TestFoo.test_bar +""", out.getvalue()) + def test_suite(): from unittest import TestLoader return TestLoader().loadTestsFromName(__name__) From c0f15af751fe031f4a7ccea93a3e4d42b443f9c3 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 30 Nov 2010 14:02:08 +1300 Subject: [PATCH 31/49] Tell testr how to use --load-list now that it exists. --- .testr.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.testr.conf b/.testr.conf index 6bf2ac6..a058ae6 100644 --- a/.testr.conf +++ b/.testr.conf @@ -1,2 +1,3 @@ [DEFAULT] -test_command=PYTHONPATH=. python -m subunit.run testtools.tests.test_suite +test_command=PYTHONPATH=. python -m subunit.run $IDOPTION testtools.tests.test_suite +test_id_option=--load-list $IDFILE From be321e8dd1da5cbaff81345556e72850bb5c157f Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Tue, 30 Nov 2010 12:54:20 +0000 Subject: [PATCH 32/49] Functional programming ftw! --- testtools/testresult/real.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testtools/testresult/real.py b/testtools/testresult/real.py index 793a9ef..758e482 100644 --- a/testtools/testresult/real.py +++ b/testtools/testresult/real.py @@ -293,8 +293,8 @@ class TextTestResult(TestResult): self.stream.write("FAILED (") details = [] details.append("failures=%d" % ( - len(self.failures) + len(self.errors) - + len(self.unexpectedSuccesses))) + sum(map(len, ( + self.failures, self.errors, self.unexpectedSuccesses))))) self.stream.write(", ".join(details)) self.stream.write(")\n") super(TextTestResult, self).stopTestRun() From d78f58b9b54b815f531afe9aecabd259da2a9fe8 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Tue, 30 Nov 2010 17:07:56 +0000 Subject: [PATCH 33/49] Add flag to make debugging twisted tests easier. --- testtools/_spinner.py | 70 ++++++++++++++++--------- testtools/deferredruntest.py | 9 +++- testtools/tests/test_deferredruntest.py | 30 +++++++++++ 3 files changed, 81 insertions(+), 28 deletions(-) diff --git a/testtools/_spinner.py b/testtools/_spinner.py index 0c30a04..eced554 100644 --- a/testtools/_spinner.py +++ b/testtools/_spinner.py @@ -20,7 +20,10 @@ __all__ = [ import signal +from testtools.monkey import MonkeyPatcher + from twisted.internet import defer +from twisted.internet.base import DelayedCall from twisted.internet.interfaces import IReactorThreads from twisted.python.failure import Failure from twisted.python.util import mergeFunctionMetadata @@ -165,13 +168,20 @@ class Spinner(object): # the ideal, and it actually works for many cases. _OBLIGATORY_REACTOR_ITERATIONS = 0 - def __init__(self, reactor): + def __init__(self, reactor, debug=False): + """Construct a Spinner. + + :param reactor: A Twisted reactor. + :param debug: Whether or not to enable Twisted's debugging. Defaults + to False. + """ self._reactor = reactor self._timeout_call = None self._success = self._UNSET self._failure = self._UNSET self._saved_signals = [] self._junk = [] + self._debug = debug def _cancel_timeout(self): if self._timeout_call: @@ -270,30 +280,38 @@ class Spinner(object): :return: Whatever is at the end of the function's callback chain. If it's an error, then raise that. """ - junk = self.get_junk() - if junk: - raise StaleJunkError(junk) - self._save_signals() - self._timeout_call = self._reactor.callLater( - timeout, self._timed_out, function, timeout) - # Calling 'stop' on the reactor will make it impossible to re-start - # the reactor. Since the default signal handlers for TERM, BREAK and - # INT all call reactor.stop(), we'll patch it over with crash. - # XXX: It might be a better idea to either install custom signal - # handlers or to override the methods that are Twisted's signal - # handlers. - stop, self._reactor.stop = self._reactor.stop, self._reactor.crash - def run_function(): - d = defer.maybeDeferred(function, *args, **kwargs) - d.addCallbacks(self._got_success, self._got_failure) - d.addBoth(self._stop_reactor) + debug = MonkeyPatcher() + if self._debug: + debug.add_patch(defer.Deferred, 'debug', True) + debug.add_patch(DelayedCall, 'debug', True) + debug.patch() try: - self._reactor.callWhenRunning(run_function) - self._reactor.run() + junk = self.get_junk() + if junk: + raise StaleJunkError(junk) + self._save_signals() + self._timeout_call = self._reactor.callLater( + timeout, self._timed_out, function, timeout) + # Calling 'stop' on the reactor will make it impossible to + # re-start the reactor. Since the default signal handlers for + # TERM, BREAK and INT all call reactor.stop(), we'll patch it over + # with crash. XXX: It might be a better idea to either install + # custom signal handlers or to override the methods that are + # Twisted's signal handlers. + stop, self._reactor.stop = self._reactor.stop, self._reactor.crash + def run_function(): + d = defer.maybeDeferred(function, *args, **kwargs) + d.addCallbacks(self._got_success, self._got_failure) + d.addBoth(self._stop_reactor) + try: + self._reactor.callWhenRunning(run_function) + self._reactor.run() + finally: + self._reactor.stop = stop + self._restore_signals() + try: + return self._get_result() + finally: + self._clean() finally: - self._reactor.stop = stop - self._restore_signals() - try: - return self._get_result() - finally: - self._clean() + debug.restore() diff --git a/testtools/deferredruntest.py b/testtools/deferredruntest.py index ec2f5de..5798828 100644 --- a/testtools/deferredruntest.py +++ b/testtools/deferredruntest.py @@ -91,7 +91,8 @@ class AsynchronousDeferredRunTest(_DeferredRunTest): This is highly experimental code. Use at your own risk. """ - def __init__(self, case, handlers=None, reactor=None, timeout=0.005): + def __init__(self, case, handlers=None, reactor=None, timeout=0.005, + debug=False): """Construct an `AsynchronousDeferredRunTest`. :param case: The `testtools.TestCase` to run. @@ -102,12 +103,16 @@ class AsynchronousDeferredRunTest(_DeferredRunTest): default reactor. :param timeout: The maximum time allowed for running a test. The default is 0.005s. + :param debug: Whether or not to enable Twisted's debugging. Use this + to get information about unhandled Deferreds and left-over + DelayedCalls. Defaults to False. """ super(AsynchronousDeferredRunTest, self).__init__(case, handlers) if reactor is None: from twisted.internet import reactor self._reactor = reactor self._timeout = timeout + self._debug = debug @classmethod def make_factory(cls, reactor=None, timeout=0.005): @@ -143,7 +148,7 @@ class AsynchronousDeferredRunTest(_DeferredRunTest): def _make_spinner(self): """Make the `Spinner` to be used to run the tests.""" - return Spinner(self._reactor) + return Spinner(self._reactor, debug=self._debug) def _run_deferred(self): """Run the test, assuming everything in it is Deferred-returning. diff --git a/testtools/tests/test_deferredruntest.py b/testtools/tests/test_deferredruntest.py index c56cccf..1124fe5 100644 --- a/testtools/tests/test_deferredruntest.py +++ b/testtools/tests/test_deferredruntest.py @@ -34,6 +34,7 @@ SynchronousDeferredRunTest = try_import( defer = try_import('twisted.internet.defer') failure = try_import('twisted.python.failure') log = try_import('twisted.python.log') +DelayedCall = try_import('twisted.internet.base.DelayedCall') class X(object): @@ -606,6 +607,35 @@ class TestAsynchronousDeferredRunTest(NeedsTwistedTestCase): error = result._events[1][2] self.assertThat(error, KeysEqual('traceback', 'twisted-log')) + def test_debugging_unchanged_during_test_by_default(self): + debugging = [(defer.Deferred.debug, DelayedCall.debug)] + class SomeCase(TestCase): + def test_debugging_enabled(self): + debugging.append((defer.Deferred.debug, DelayedCall.debug)) + test = SomeCase('test_debugging_enabled') + runner = AsynchronousDeferredRunTest( + test, handlers=test.exception_handlers, + reactor=self.make_reactor(), timeout=self.make_timeout()) + runner.run(self.make_result()) + self.assertEqual(debugging[0], debugging[1]) + + def test_debugging_enabled_during_test_with_debug_flag(self): + self.patch(defer.Deferred, 'debug', False) + self.patch(DelayedCall, 'debug', False) + debugging = [] + class SomeCase(TestCase): + def test_debugging_enabled(self): + debugging.append((defer.Deferred.debug, DelayedCall.debug)) + test = SomeCase('test_debugging_enabled') + runner = AsynchronousDeferredRunTest( + test, handlers=test.exception_handlers, + reactor=self.make_reactor(), timeout=self.make_timeout(), + debug=True) + runner.run(self.make_result()) + self.assertEqual([(True, True)], debugging) + self.assertEqual(False, defer.Deferred.debug) + self.assertEqual(False, defer.Deferred.debug) + class TestAssertFailsWith(NeedsTwistedTestCase): """Tests for `assert_fails_with`.""" From 693836a78954fec187e9ef18226f63122e74de33 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Tue, 30 Nov 2010 17:13:22 +0000 Subject: [PATCH 34/49] Make debug available from the factory. --- testtools/deferredruntest.py | 4 ++-- testtools/tests/test_deferredruntest.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/testtools/deferredruntest.py b/testtools/deferredruntest.py index 5798828..50153be 100644 --- a/testtools/deferredruntest.py +++ b/testtools/deferredruntest.py @@ -115,14 +115,14 @@ class AsynchronousDeferredRunTest(_DeferredRunTest): self._debug = debug @classmethod - def make_factory(cls, reactor=None, timeout=0.005): + def make_factory(cls, reactor=None, timeout=0.005, debug=False): """Make a factory that conforms to the RunTest factory interface.""" # This is horrible, but it means that the return value of the method # will be able to be assigned to a class variable *and* also be # invoked directly. class AsynchronousDeferredRunTestFactory: def __call__(self, case, handlers=None): - return cls(case, handlers, reactor, timeout) + return cls(case, handlers, reactor, timeout, debug) return AsynchronousDeferredRunTestFactory() @defer.deferredGenerator diff --git a/testtools/tests/test_deferredruntest.py b/testtools/tests/test_deferredruntest.py index 1124fe5..04614df 100644 --- a/testtools/tests/test_deferredruntest.py +++ b/testtools/tests/test_deferredruntest.py @@ -492,6 +492,17 @@ class TestAsynchronousDeferredRunTest(NeedsTwistedTestCase): self.assertIs(self, runner.case) self.assertEqual([handler], runner.handlers) + def test_convenient_construction_default_debugging(self): + # As a convenience method, AsynchronousDeferredRunTest has a + # classmethod that returns an AsynchronousDeferredRunTest + # factory. This factory has the same API as the RunTest constructor. + handler = object() + factory = AsynchronousDeferredRunTest.make_factory(debug=True) + runner = factory(self, [handler]) + self.assertIs(self, runner.case) + self.assertEqual([handler], runner.handlers) + self.assertEqual(True, runner._debug) + def test_deferred_error(self): class SomeTest(TestCase): def test_something(self): From 2453382d7e3d8654d867f327c1ac75459e42c2ce Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Sun, 5 Dec 2010 10:04:26 +1300 Subject: [PATCH 35/49] Fix the regression caused when fixing bug 654474 by permitting decorated test results to set their own policy for what constitutes and error and what doesn't. (Robert Collins, #683332) --- NEWS | 12 +++++++++-- testtools/testresult/real.py | 27 +++++++++++-------------- testtools/tests/test_testresult.py | 32 ++++++++++++++++++++++-------- 3 files changed, 46 insertions(+), 25 deletions(-) diff --git a/NEWS b/NEWS index dcabfbd..5519308 100644 --- a/NEWS +++ b/NEWS @@ -10,10 +10,18 @@ Changes * addUnexpectedSuccess is translated to addFailure for test results that don't know about addUnexpectedSuccess. Further, it fails the entire result for all testtools TestResults (i.e. wasSuccessful() returns False after - addUnexpectedSuccess has been called). (Jonathan Lange, #654474) + addUnexpectedSuccess has been called). Note that when using a delegating + result such as ThreadsafeForwardingResult, MultiTestResult or + ExtendedToOriginalDecorator then the behaviour of addUnexpectedSuccess is + determined by the delegated to result(s). + (Jonathan Lange, Robert Collins, #654474, #683332) * startTestRun will reset any errors on the result. That is, wasSuccessful() - will always return True immediately after startTestRun() is called. + will always return True immediately after startTestRun() is called. This + only applies to delegated test results (ThreadsafeForwardingResult, + MultiTestResult and ExtendedToOriginalDecorator) if the delegated to result + is a testtools test result - we cannot reliably reset the state of unknown + test result class instances. (Jonathan Lange, Robert Collins, #683332) * Responsibility for running test cleanups has been moved to ``RunTest``. This change does not affect public APIs and can be safely ignored by test diff --git a/testtools/testresult/real.py b/testtools/testresult/real.py index 758e482..c57dbdd 100644 --- a/testtools/testresult/real.py +++ b/testtools/testresult/real.py @@ -35,13 +35,11 @@ class TestResult(unittest.TestResult): """ def __init__(self): - super(TestResult, self).__init__() - self.skip_reasons = {} - self.__now = None - # -- Start: As per python 2.7 -- - self.expectedFailures = [] - self.unexpectedSuccesses = [] - # -- End: As per python 2.7 -- + # startTestRun resets all attributes, and older clients don't know to + # call startTestRun, so it is called once here. + # Because subclasses may reasonably not expect this, we call the + # specific version we want to run. + TestResult.startTestRun(self) def addExpectedFailure(self, test, err=None, details=None): """Called when a test has failed in an expected manner. @@ -160,9 +158,13 @@ class TestResult(unittest.TestResult): New in python 2.7 """ + super(TestResult, self).__init__() + self.skip_reasons = {} + self.__now = None + # -- Start: As per python 2.7 -- + self.expectedFailures = [] self.unexpectedSuccesses = [] - self.errors = [] - self.failures = [] + # -- End: As per python 2.7 -- def stopTestRun(self): """Called after a test run completes @@ -406,13 +408,11 @@ class ExtendedToOriginalDecorator(object): def __init__(self, decorated): self.decorated = decorated - self._was_successful = True def __getattr__(self, name): return getattr(self.decorated, name) def addError(self, test, err=None, details=None): - self._was_successful = False self._check_args(err, details) if details is not None: try: @@ -437,7 +437,6 @@ class ExtendedToOriginalDecorator(object): return addExpectedFailure(test, err) def addFailure(self, test, err=None, details=None): - self._was_successful = False self._check_args(err, details) if details is not None: try: @@ -464,7 +463,6 @@ class ExtendedToOriginalDecorator(object): return addSkip(test, reason) def addUnexpectedSuccess(self, test, details=None): - self._was_successful = False outcome = getattr(self.decorated, 'addUnexpectedSuccess', None) if outcome is None: try: @@ -521,7 +519,6 @@ class ExtendedToOriginalDecorator(object): return self.decorated.startTest(test) def startTestRun(self): - self._was_successful = True try: return self.decorated.startTestRun() except AttributeError: @@ -552,7 +549,7 @@ class ExtendedToOriginalDecorator(object): return method(a_datetime) def wasSuccessful(self): - return self._was_successful + return self.decorated.wasSuccessful() class _StringException(Exception): diff --git a/testtools/tests/test_testresult.py b/testtools/tests/test_testresult.py index 0414109..a0e090d 100644 --- a/testtools/tests/test_testresult.py +++ b/testtools/tests/test_testresult.py @@ -132,7 +132,7 @@ class Python27Contract(Python26Contract): result.stopTestRun() -class TestResultContract(Python27Contract): +class DetailsContract(Python27Contract): """Tests for the contract of TestResults.""" def test_addExpectedFailure_details(self): @@ -171,6 +171,13 @@ class TestResultContract(Python27Contract): result.startTest(self) result.addSuccess(self, details={}) + +class FallbackContract(DetailsContract): + """When we fallback we take our policy choice to map calls. + + For instance, we map unexpectedSuccess to an error code, not to success. + """ + def test_addUnexpectedSuccess_was_successful(self): # addUnexpectedSuccess fails test run in testtools. result = self.makeResult() @@ -179,6 +186,14 @@ class TestResultContract(Python27Contract): result.stopTest(self) self.assertFalse(result.wasSuccessful()) + +class StartTestRunContract(FallbackContract): + """Defines the contract for testtools policy choices. + + That is things which are not simply extensions to unittest but choices we + have made differently. + """ + def test_startTestRun_resets_unexpected_success(self): result = self.makeResult() result.startTest(self) @@ -203,25 +218,26 @@ class TestResultContract(Python27Contract): result.startTestRun() self.assertTrue(result.wasSuccessful()) -class TestTestResultContract(TestCase, TestResultContract): + +class TestTestResultContract(TestCase, StartTestRunContract): def makeResult(self): return TestResult() -class TestMultiTestResultContract(TestCase, TestResultContract): +class TestMultiTestResultContract(TestCase, StartTestRunContract): def makeResult(self): return MultiTestResult(TestResult(), TestResult()) -class TestTextTestResultContract(TestCase, TestResultContract): +class TestTextTestResultContract(TestCase, StartTestRunContract): def makeResult(self): return TextTestResult(StringIO()) -class TestThreadSafeForwardingResultContract(TestCase, TestResultContract): +class TestThreadSafeForwardingResultContract(TestCase, StartTestRunContract): def makeResult(self): result_semaphore = threading.Semaphore(1) @@ -229,7 +245,7 @@ class TestThreadSafeForwardingResultContract(TestCase, TestResultContract): return ThreadsafeForwardingResult(target, result_semaphore) -class TestExtendedTestResultContract(TestCase, TestResultContract): +class TestExtendedTestResultContract(TestCase, StartTestRunContract): def makeResult(self): return ExtendedTestResult() @@ -241,7 +257,7 @@ class TestPython26TestResultContract(TestCase, Python26Contract): return Python26TestResult() -class TestAdaptedPython26TestResultContract(TestCase, TestResultContract): +class TestAdaptedPython26TestResultContract(TestCase, FallbackContract): def makeResult(self): return ExtendedToOriginalDecorator(Python26TestResult()) @@ -253,7 +269,7 @@ class TestPython27TestResultContract(TestCase, Python27Contract): return Python27TestResult() -class TestAdaptedPython27TestResultContract(TestCase, TestResultContract): +class TestAdaptedPython27TestResultContract(TestCase, DetailsContract): def makeResult(self): return ExtendedToOriginalDecorator(Python27TestResult()) From 16ab833dd6c873dae3305adddcbacc449f9f0d9a Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Mon, 6 Dec 2010 12:58:48 +1300 Subject: [PATCH 36/49] Explain the startTestRun behaviour in testtools. --- testtools/testresult/real.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/testtools/testresult/real.py b/testtools/testresult/real.py index c57dbdd..d1a1023 100644 --- a/testtools/testresult/real.py +++ b/testtools/testresult/real.py @@ -156,7 +156,8 @@ class TestResult(unittest.TestResult): def startTestRun(self): """Called before a test run starts. - New in python 2.7 + New in python 2.7. The testtools version resets the result to a + pristine condition ready for use in another test run. """ super(TestResult, self).__init__() self.skip_reasons = {} From 197cb2e259492218f569b27be4a00944882d44b5 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Mon, 6 Dec 2010 20:25:09 +1300 Subject: [PATCH 37/49] Use shiny new parallel test facility for testr. --- .testr.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.testr.conf b/.testr.conf index a058ae6..12d6685 100644 --- a/.testr.conf +++ b/.testr.conf @@ -1,3 +1,4 @@ [DEFAULT] -test_command=PYTHONPATH=. python -m subunit.run $IDOPTION testtools.tests.test_suite +test_command=PYTHONPATH=. python -m subunit.run $LISTOPT $IDOPTION testtools.tests.test_suite test_id_option=--load-list $IDFILE +test_list_option=--list From 24692e0dc71e795f9524420e37d777bda330cabd Mon Sep 17 00:00:00 2001 From: Jelmer Vernooij Date: Tue, 7 Dec 2010 13:11:13 +0100 Subject: [PATCH 38/49] Let stdout default to sys.stdout in TestProgram. --- testtools/run.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testtools/run.py b/testtools/run.py index da4496a..272992c 100755 --- a/testtools/run.py +++ b/testtools/run.py @@ -132,6 +132,8 @@ class TestProgram(object): self.module = module if argv is None: argv = sys.argv + if stdout is None: + stdout = sys.stdout self.exit = exit self.failfast = failfast From e8632e68794c87044ec1ee075cbf2861cf5377e2 Mon Sep 17 00:00:00 2001 From: Martin Date: Fri, 10 Dec 2010 23:38:16 +0000 Subject: [PATCH 39/49] Make use of StringIO in testtools.tests.test_run Python 3 compatible --- testtools/tests/test_run.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/testtools/tests/test_run.py b/testtools/tests/test_run.py index 5087527..26e23fe 100644 --- a/testtools/tests/test_run.py +++ b/testtools/tests/test_run.py @@ -2,10 +2,9 @@ """Tests for the test runner logic.""" -import StringIO - -from testtools.helpers import try_import +from testtools.helpers import try_import, try_imports fixtures = try_import('fixtures') +StringIO = try_imports(['StringIO.StringIO', 'io.StringIO']) import testtools from testtools import TestCase, run @@ -43,7 +42,7 @@ class TestRun(TestCase): if fixtures is None: self.skipTest("Need fixtures") package = self.useFixture(SampleTestFixture()) - out = StringIO.StringIO() + out = StringIO() run.main(['prog', '-l', 'testtools.runexample.test_suite'], out) self.assertEqual("""testtools.runexample.TestFoo.test_bar testtools.runexample.TestFoo.test_quux From 90ca40189de9a88c25f306f706e6bfcdf603f4f1 Mon Sep 17 00:00:00 2001 From: Martin Date: Sat, 11 Dec 2010 00:07:01 +0000 Subject: [PATCH 40/49] Expose 'all' from compat and implement it for Python 2.4 --- testtools/compat.py | 7 +++++++ testtools/testresult/real.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/testtools/compat.py b/testtools/compat.py index 5c2b224..ecbfb42 100644 --- a/testtools/compat.py +++ b/testtools/compat.py @@ -66,12 +66,19 @@ _u.__doc__ = __u_doc if sys.version_info > (2, 5): + all = all _error_repr = BaseException.__repr__ def isbaseexception(exception): """Return whether exception inherits from BaseException only""" return (isinstance(exception, BaseException) and not isinstance(exception, Exception)) else: + def all(iterable): + """If contents of iterable all evaluate as boolean True""" + for obj in iterable: + if not obj: + return False + return True def _error_repr(exception): """Format an exception instance as Python 2.5 and later do""" return exception.__class__.__name__ + repr(exception.args) diff --git a/testtools/testresult/real.py b/testtools/testresult/real.py index d1a1023..d2e8213 100644 --- a/testtools/testresult/real.py +++ b/testtools/testresult/real.py @@ -14,7 +14,7 @@ import datetime import sys import unittest -from testtools.compat import _format_exc_info, str_is_unicode, _u +from testtools.compat import all, _format_exc_info, str_is_unicode, _u class TestResult(unittest.TestResult): From 6589b3668cc9328661ad59720b9cf0b6c2520357 Mon Sep 17 00:00:00 2001 From: Jelmer Vernooij Date: Sun, 12 Dec 2010 05:11:39 +0100 Subject: [PATCH 41/49] Support optional 'msg' argument to TestCase.assertIsInstance. --- testtools/testcase.py | 9 +++++---- testtools/tests/test_testtools.py | 10 ++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/testtools/testcase.py b/testtools/testcase.py index ba7b480..804684a 100644 --- a/testtools/testcase.py +++ b/testtools/testcase.py @@ -300,10 +300,11 @@ class TestCase(unittest.TestCase): self.assertTrue( needle not in haystack, '%r in %r' % (needle, haystack)) - def assertIsInstance(self, obj, klass): - self.assertTrue( - isinstance(obj, klass), - '%r is not an instance of %s' % (obj, self._formatTypes(klass))) + def assertIsInstance(self, obj, klass, msg=None): + if msg is None: + msg = '%r is not an instance of %s' % ( + obj, self._formatTypes(klass)) + self.assertTrue(isinstance(obj, klass), msg) def assertRaises(self, excClass, callableObj, *args, **kwargs): """Fail unless an exception of class excClass is thrown diff --git a/testtools/tests/test_testtools.py b/testtools/tests/test_testtools.py index 2845730..d0b3ae7 100644 --- a/testtools/tests/test_testtools.py +++ b/testtools/tests/test_testtools.py @@ -375,6 +375,16 @@ class TestAssertions(TestCase): '42 is not an instance of %s' % self._formatTypes([Foo, Bar]), self.assertIsInstance, 42, (Foo, Bar)) + def test_assertIsInstance_overridden_message(self): + # assertIsInstance(obj, klass, msg) fails the test with the specified + # message when obj is not an instance of klass. + + class Foo(object): + """Simple class for testing assertIsInstance.""" + + self.assertFails("Bericht", + self.assertIsInstance, 42, Foo, "Bericht") + def test_assertIs(self): # assertIs asserts that an object is identical to another object. self.assertIs(None, None) From 41d560d5898e51739354d59f9e65de31d864b20a Mon Sep 17 00:00:00 2001 From: Michael Hudson Date: Mon, 13 Dec 2010 12:41:57 +1300 Subject: [PATCH 42/49] trivial test fix --- testtools/tests/test_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testtools/tests/test_run.py b/testtools/tests/test_run.py index 26e23fe..8f88fb6 100644 --- a/testtools/tests/test_run.py +++ b/testtools/tests/test_run.py @@ -52,7 +52,7 @@ testtools.runexample.TestFoo.test_quux if fixtures is None: self.skipTest("Need fixtures") package = self.useFixture(SampleTestFixture()) - out = StringIO.StringIO() + out = StringIO() # We load two tests - one that exists and one that doesn't, and we # should get the one that exists and neither the one that doesn't nor # the unmentioned one that does. From 00ebb7fd6891a7c5a275f53911d69c66531be29a Mon Sep 17 00:00:00 2001 From: Michael Hudson Date: Mon, 13 Dec 2010 12:42:15 +1300 Subject: [PATCH 43/49] ignore .testrepository --- .bzrignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.bzrignore b/.bzrignore index 8b9c8dc..f72bf52 100644 --- a/.bzrignore +++ b/.bzrignore @@ -6,3 +6,4 @@ tags TAGS apidocs _trial_temp +./.testrepository From a4ed9320c42e2ddee9b9e5a7ba1ca0939fb9671c Mon Sep 17 00:00:00 2001 From: Michael Hudson Date: Mon, 13 Dec 2010 12:48:10 +1300 Subject: [PATCH 44/49] remove the final newline from the description of MismatchesAll --- testtools/matchers.py | 2 +- testtools/tests/test_matchers.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/testtools/matchers.py b/testtools/matchers.py index d8cca2c..06b348c 100644 --- a/testtools/matchers.py +++ b/testtools/matchers.py @@ -316,7 +316,7 @@ class MismatchesAll(Mismatch): descriptions = ["Differences: ["] for mismatch in self.mismatches: descriptions.append(mismatch.describe()) - descriptions.append("]\n") + descriptions.append("]") return '\n'.join(descriptions) diff --git a/testtools/tests/test_matchers.py b/testtools/tests/test_matchers.py index cd684a0..bbcd87e 100644 --- a/testtools/tests/test_matchers.py +++ b/testtools/tests/test_matchers.py @@ -247,8 +247,7 @@ Expected: Got: 3 -] -""", +]""", "3", MatchesAny(DocTestMatches("1"), DocTestMatches("2")))] @@ -264,8 +263,7 @@ class TestMatchesAllInterface(TestCase, TestMatchersInterface): describe_examples = [("""Differences: [ 1 == 1 -] -""", +]""", 1, MatchesAll(NotEquals(1), NotEquals(2)))] From 0694f37bbb9ce0956cafdbf578bc69fc77ef7965 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 14 Dec 2010 09:31:14 +1300 Subject: [PATCH 45/49] Avoid race condition in test_spinner.TestRunInReactor.test_clean_running_threads due to twisted threadpool not joining stopped workers. --- testtools/_spinner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/testtools/_spinner.py b/testtools/_spinner.py index eced554..98b51a6 100644 --- a/testtools/_spinner.py +++ b/testtools/_spinner.py @@ -232,7 +232,6 @@ class Spinner(object): # we aren't going to bother. junk.append(selectable) if IReactorThreads.providedBy(self._reactor): - self._reactor.suggestThreadPoolSize(0) if self._reactor.threadpool is not None: self._reactor._stopThreadPool() self._junk.extend(junk) From ba8af964e7e2b7378e21874c4383870b82bd1599 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 14 Dec 2010 11:49:43 +1300 Subject: [PATCH 46/49] Workaround 2.4 threading bug in test_spinner. --- testtools/tests/test_spinner.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/testtools/tests/test_spinner.py b/testtools/tests/test_spinner.py index f898956..5c6139d 100644 --- a/testtools/tests/test_spinner.py +++ b/testtools/tests/test_spinner.py @@ -244,7 +244,14 @@ class TestRunInReactor(NeedsTwistedTestCase): timeout = self.make_timeout() spinner = self.make_spinner(reactor) spinner.run(timeout, reactor.callInThread, time.sleep, timeout / 2.0) - self.assertThat(list(threading.enumerate()), Equals(current_threads)) + # Python before 2.5 has a race condition with thread handling where + # join() does not remove threads from enumerate before returning - the + # thread being joined does the removal. This was fixed in Python 2.5 + # but we still support 2.4, so we have to workaround the issue. + # http://bugs.python.org/issue1703448. + self.assertThat( + [thread for thread in threading.enumerate() if thread.isAlive()], + Equals(current_threads)) def test_leftover_junk_available(self): # If 'run' is given a function that leaves the reactor dirty in some From 2234442dd0bab8faec0aa9c079317f53833b457f Mon Sep 17 00:00:00 2001 From: Martin Date: Thu, 16 Dec 2010 20:18:51 +0000 Subject: [PATCH 47/49] Avoid issue with broken __unicode__ method on some Python 2.6 minor versions --- testtools/tests/test_testresult.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/testtools/tests/test_testresult.py b/testtools/tests/test_testresult.py index a0e090d..cad8c75 100644 --- a/testtools/tests/test_testresult.py +++ b/testtools/tests/test_testresult.py @@ -1230,6 +1230,8 @@ class TestNonAsciiResults(TestCase): "class UnprintableError(Exception):\n" " def __str__(self):\n" " raise RuntimeError\n" + " def __unicode__(self):\n" + " raise RuntimeError\n" " def __repr__(self):\n" " raise RuntimeError\n") textoutput = self._test_external_case( From 0687430d00d49781dac541da89476114b0d58f3e Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Sat, 18 Dec 2010 20:06:10 +1300 Subject: [PATCH 48/49] Release 0.9.8. --- NEWS | 14 ++++++++++++-- testtools/__init__.py | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 42bb311..fd2f8bf 100644 --- a/NEWS +++ b/NEWS @@ -1,8 +1,18 @@ testtools NEWS ++++++++++++++ -NEXT -~~~~ +0.9.8 +~~~~~ + +In this release we bring some very interesting improvements: + +* new matchers for exceptions, sets, lists, dicts and more. + +* experimental (works but the contract isn't supported) twisted reactor + support. + +* The built in runner can now list tests and filter tests (the -l and + --load-list options). Changes ------- diff --git a/testtools/__init__.py b/testtools/__init__.py index 0f85426..52d3945 100644 --- a/testtools/__init__.py +++ b/testtools/__init__.py @@ -69,4 +69,4 @@ from testtools.testsuite import ( # If the releaselevel is 'final', then the tarball will be major.minor.micro. # Otherwise it is major.minor.micro~$(revno). -__version__ = (0, 9, 8, 'dev', 0) +__version__ = (0, 9, 8, 'final', 0) From c777b9da74f6259b0153fe7faeccb871e3a54eef Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Mon, 20 Dec 2010 08:03:10 +1300 Subject: [PATCH 49/49] Changes ------- * The timestamps generated by ``TestResult`` objects when no timing data has been received are now datetime-with-timezone, which allows them to be sensibly serialised and transported. (Robert Collins, #692297) Improvements ------------ * ``MultiTestResult`` now forwards the ``time`` API. (Robert Collins, #692294) --- NEWS | 12 ++++++++++++ testtools/testresult/real.py | 24 +++++++++++++++++++++++- testtools/tests/test_testresult.py | 14 ++++++++++---- 3 files changed, 45 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index f09a460..4d2a744 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,18 @@ testtools NEWS NEXT ~~~~ +Changes +------- + +* The timestamps generated by ``TestResult`` objects when no timing data has + been received are now datetime-with-timezone, which allows them to be + sensibly serialised and transported. (Robert Collins, #692297) + +Improvements +------------ + +* ``MultiTestResult`` now forwards the ``time`` API. (Robert Collins, #692294) + 0.9.8 ~~~~~ diff --git a/testtools/testresult/real.py b/testtools/testresult/real.py index d2e8213..b521251 100644 --- a/testtools/testresult/real.py +++ b/testtools/testresult/real.py @@ -16,6 +16,25 @@ import unittest from testtools.compat import all, _format_exc_info, str_is_unicode, _u +# From http://docs.python.org/library/datetime.html +_ZERO = datetime.timedelta(0) + +# A UTC class. + +class UTC(datetime.tzinfo): + """UTC""" + + def utcoffset(self, dt): + return _ZERO + + def tzname(self, dt): + return "UTC" + + def dst(self, dt): + return _ZERO + +utc = UTC() + class TestResult(unittest.TestResult): """Subclass of unittest.TestResult extending the protocol for flexability. @@ -149,7 +168,7 @@ class TestResult(unittest.TestResult): time() method. """ if self.__now is None: - return datetime.datetime.now() + return datetime.datetime.now(utc) else: return self.__now @@ -238,6 +257,9 @@ class MultiTestResult(TestResult): def stopTestRun(self): return self._dispatch('stopTestRun') + def time(self, a_datetime): + return self._dispatch('time', a_datetime) + def done(self): return self._dispatch('done') diff --git a/testtools/tests/test_testresult.py b/testtools/tests/test_testresult.py index cad8c75..57c3293 100644 --- a/testtools/tests/test_testresult.py +++ b/testtools/tests/test_testresult.py @@ -45,6 +45,7 @@ from testtools.tests.helpers import ( ExtendedTestResult, an_exc_info ) +from testtools.testresult.real import utc StringIO = try_imports(['StringIO.StringIO', 'io.StringIO']) @@ -305,10 +306,10 @@ class TestTestResult(TestCase): self.addCleanup(restore) class Module: pass - now = datetime.datetime.now() + now = datetime.datetime.now(utc) stubdatetime = Module() stubdatetime.datetime = Module() - stubdatetime.datetime.now = lambda: now + stubdatetime.datetime.now = lambda tz: now testresult.real.datetime = stubdatetime # Calling _now() looks up the time. self.assertEqual(now, result._now()) @@ -323,7 +324,7 @@ class TestTestResult(TestCase): def test_now_datetime_time(self): result = self.makeResult() - now = datetime.datetime.now() + now = datetime.datetime.now(utc) result.time(now) self.assertEqual(now, result._now()) @@ -424,6 +425,11 @@ class TestMultiTestResult(TestWithFakeExceptions): result = multi_result.stopTestRun() self.assertEqual(('foo', 'foo'), result) + def test_time(self): + # the time call is dispatched, not eaten by the base class + self.multiResult.time('foo') + self.assertResultLogsEqual([('time', 'foo')]) + class TestTextTestResult(TestCase): """Tests for `TextTestResult`.""" @@ -501,7 +507,7 @@ class TestTextTestResult(TestCase): def test_stopTestRun_current_time(self): test = self.make_test() - now = datetime.datetime.now() + now = datetime.datetime.now(utc) self.result.time(now) self.result.startTestRun() self.result.startTest(test)