diff --git a/gabbi/suite.py b/gabbi/suite.py index 765a929..f6d3868 100644 --- a/gabbi/suite.py +++ b/gabbi/suite.py @@ -16,6 +16,7 @@ This suite has two features: the contained tests are ordered and there are suite-level fixtures that operate as context managers. """ +import sys import unittest from wsgi_intercept import interceptor @@ -58,6 +59,26 @@ class GabbiSuite(unittest.TestSuite): except unittest.SkipTest as exc: for test in self._tests: result.addSkip(test, str(exc)) + # If we have an exception in the nested fixtures, that means + # there's been an exception somewhere in the cycle other + # than a specific test (as that would have been caught + # already), thus from a fixture. If that exception were to + # continue to raise here, then some test runners would + # swallow it and the traceback of the failure would be + # undiscoverable. To ensure the traceback is reported (via + # the testrunner) to a human, the first test in the suite is + # marked as having an error (it's fixture failed) and then + # the entire suite is skipped, and the result stream told + # we're done. If there are no tests (an empty suite) the + # exception is re-raised. + except Exception as exc: + if self._tests: + result.addError(self._tests[0], sys.exc_info()) + for test in self._tests: + result.addSkip(test, 'fixture failure') + result.stop() + else: + raise return result diff --git a/gabbi/tests/test_suite.py b/gabbi/tests/test_suite.py new file mode 100644 index 0000000..eb1bd22 --- /dev/null +++ b/gabbi/tests/test_suite.py @@ -0,0 +1,54 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Unit tests for the gabbi.suite. +""" + +import sys +import unittest + +from gabbi import fixture +from gabbi import suitemaker + +VALUE_ERROR = 'value error sentinel' + + +class FakeFixture(fixture.GabbiFixture): + + def start_fixture(self): + raise ValueError(VALUE_ERROR) + + +class SuiteTest(unittest.TestCase): + + def test_suite_catches_fixture_fail(self): + """When a fixture fails in start_fixture it should fail + the first test in the suite and skip the others. + """ + loader = unittest.defaultTestLoader + result = unittest.TestResult() + test_data = {'fixtures': ['FakeFixture'], + 'tests': [{'name': 'alpha', 'GET': '/'}, + {'name': 'beta', 'GET': '/'}]} + test_suite = suitemaker.test_suite_from_dict( + loader, 'foo', test_data, '.', 'localhost', + 80, sys.modules[__name__], None) + + test_suite.run(result) + + self.assertEqual(2, len(result.skipped)) + self.assertEqual(1, len(result.errors)) + + errored_test, trace = result.errors[0] + + self.assertIn('foo_alpha', str(errored_test)) + self.assertIn(VALUE_ERROR, trace)