Handle exceptions when starting fixtures

Some test runners will swallow exceptions and tracebacks that happen
during the start_fixture method of a Fixture. This can lead to a
great deal of confusion when creating new fixtures. This change
ensures that the traceback is made available by forcing an error to
be reported on the first test in the test suite using the fixture.
Additional detail can be found in the comment associated with the
change.

The added test_suite file tests this change by creating a suite with
two tests that use a fixture that will raise an exception in
start_fixture. The test confirms that the right test is errored,
that both are skipped and that the traceback includes expected
information.

Fixes #199

gabbi/suite.py

@@ -16,6 +16,7 @@ This suite has two features: the contained tests are ordered and there
 are suite-level fixtures that operate as context managers.
 """
 
+import sys
 import unittest
 
 from wsgi_intercept import interceptor
@@ -58,6 +59,26 @@ class GabbiSuite(unittest.TestSuite):
         except unittest.SkipTest as exc:
             for test in self._tests:
                 result.addSkip(test, str(exc))
+        # If we have an exception in the nested fixtures, that means
+        # there's been an exception somewhere in the cycle other
+        # than a specific test (as that would have been caught
+        # already), thus from a fixture. If that exception were to
+        # continue to raise here, then some test runners would
+        # swallow it and the traceback of the failure would be
+        # undiscoverable. To ensure the traceback is reported (via
+        # the testrunner) to a human, the first test in the suite is
+        # marked as having an error (it's fixture failed) and then
+        # the entire suite is skipped, and the result stream told
+        # we're done. If there are no tests (an empty suite) the
+        # exception is re-raised.
+        except Exception as exc:
+            if self._tests:
+                result.addError(self._tests[0], sys.exc_info())
+                for test in self._tests:
+                    result.addSkip(test, 'fixture failure')
+                result.stop()
+            else:
+                raise
 
         return result
 

gabbi/tests/test_suite.py

@@ -0,0 +1,54 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Unit tests for the gabbi.suite.
+"""
+
+import sys
+import unittest
+
+from gabbi import fixture
+from gabbi import suitemaker
+
+VALUE_ERROR = 'value error sentinel'
+
+
+class FakeFixture(fixture.GabbiFixture):
+
+    def start_fixture(self):
+        raise ValueError(VALUE_ERROR)
+
+
+class SuiteTest(unittest.TestCase):
+
+    def test_suite_catches_fixture_fail(self):
+        """When a fixture fails in start_fixture it should fail
+        the first test in the suite and skip the others.
+        """
+        loader = unittest.defaultTestLoader
+        result = unittest.TestResult()
+        test_data = {'fixtures': ['FakeFixture'],
+                     'tests': [{'name': 'alpha', 'GET': '/'},
+                               {'name': 'beta', 'GET': '/'}]}
+        test_suite = suitemaker.test_suite_from_dict(
+            loader, 'foo', test_data, '.', 'localhost',
+            80, sys.modules[__name__], None)
+
+        test_suite.run(result)
+
+        self.assertEqual(2, len(result.skipped))
+        self.assertEqual(1, len(result.errors))
+
+        errored_test, trace = result.errors[0]
+
+        self.assertIn('foo_alpha', str(errored_test))
+        self.assertIn(VALUE_ERROR, trace)