Fix pytest failure handling

Fixes #124

Thought pytest was collecting and running tests, the results were
not actually being handled. If a test failed, it still appeared to
pass. The fundamental reason for this is that only the GabbiSuite
which contains all the tests from each YAML file was being checked.
These look like tests to pytest and pass.

The eventual fix is fairly complex and could maybe be made less so
by learning how to use modern parameterized pytest[1] rather than the
old yield style being used here. The fix includes:

* Creating a PyTestResult class which translates various unitest
  result types into pytest skips or xfails or reraises errors as
  required.

* Works around the GabbiSuite.run() based fixture handling that
  unitest-based runners automatically use but won't work properly
  in the pytest yield setting by adding start() and stop() methods
  the suite and yielding artifical tests which call start and stop.[2]

Besides getting failing tests to actually fail this also gets some
other features working:

* xfail and skip work, including skipping an entire yaml file with
  he SkipFixture
* If a single test from a file is selected, all the prior tests in
  the file will run too, but only the one requested will report.

[1] http://pytest.org/latest/parametrize.html#pytest-generate-tests

[2] This causes the number of tests to increase but it seems to be the
only way to get things working without larger changes.

gabbi/driver.py

@@ -38,6 +38,7 @@ import yaml
 from gabbi import case
 from gabbi import handlers
 from gabbi import httpclient
+from gabbi import reporter
 from gabbi import suite as gabbi_suite
 
 
@@ -243,6 +244,7 @@ def py_test_generator(test_dir, host=None, port=8001, intercept=None,
     a way that pytest can handle.
     """
     loader = unittest.TestLoader()
+    result = reporter.PyTestResult()
     tests = build_tests(test_dir, loader, host=host, port=port,
                         intercept=intercept,
                         test_loader_name=test_loader_name,
@@ -252,10 +254,12 @@ def py_test_generator(test_dir, host=None, port=8001, intercept=None,
 
     for test in tests:
         if hasattr(test, '_tests'):
-            for subtest in test._tests:
-                yield '%s' % subtest.__class__.__name__, subtest
-        else:
-            yield '%s' % test.__class__.__name__, test
+            # Establish fixtures as if they were tests.
+            yield 'start_%s' % test._tests[0].__class__.__name__, \
+                test.start, result
+            for subtest in test:
+                yield '%s' % subtest.__class__.__name__, subtest, result
+            yield 'stop_%s' % test._tests[0].__class__.__name__, test.stop
 
 
 def load_yaml(yaml_file):

gabbi/reporter.py

@@ -13,9 +13,12 @@
 # under the License.
 """TestRunner and TestResult for gabbi-run."""
 
+from unittest import TestResult
 from unittest import TextTestResult
 from unittest import TextTestRunner
 
+import pytest
+
 from gabbi import utils
 
 
@@ -100,6 +103,21 @@ class ConciseTestResult(TextTestResult):
                 self.stream.writeln('\t%s' % line)
 
 
+class PyTestResult(TestResult):
+
+    def addFailure(self, test, err):
+        raise err[1]
+
+    def addError(self, test, err):
+        raise err[1]
+
+    def addSkip(self, test, reason):
+        pytest.skip(reason)
+
+    def addExpectedFailure(self, test, err):
+        pytest.xfail('%s' % err[1])
+
+
 class ConciseTestRunner(TextTestRunner):
     """A TextTestRunner that uses ConciseTestResult for reporting results."""
     resultclass = ConciseTestResult

gabbi/suite.py

@@ -24,6 +24,11 @@ from wsgi_intercept import interceptor
 from gabbi import fixture
 
 
+def noop(*args):
+    """A noop method used to disable collected tests."""
+    pass
+
+
 class GabbiSuite(suite.TestSuite):
     """A TestSuite with fixtures.
 
@@ -34,17 +39,60 @@ class GabbiSuite(suite.TestSuite):
     tests in this suite will be skipped.
     """
 
-    def run(self, result, debug=False):
+    def run(self, result, debug=False, pytest=False):
         """Override TestSuite run to start suite-level fixtures.
 
         To avoid exception confusion, use a null Fixture when there
         are no fixtures.
         """
 
-        # If there are fixtures, nest in their context.
-        fixtures = [fixture.GabbiFixture]
-        intercept = None
+        fixtures, intercept, host, port, prefix = self._get_intercept()
 
+        try:
+            with fixture.nest([fix() for fix in fixtures]):
+                if intercept:
+                    with interceptor.Urllib3Interceptor(
+                            intercept, host, port, prefix):
+                        result = super(GabbiSuite, self).run(result, debug)
+                else:
+                    result = super(GabbiSuite, self).run(result, debug)
+        except case.SkipTest as exc:
+            for test in self._tests:
+                result.addSkip(test, str(exc))
+
+        return result
+
+    def start(self, result):
+        """Start fixtures when using pytest."""
+        fixtures, intercept, host, port, prefix = self._get_intercept()
+
+        self.used_fixtures = []
+        try:
+            for fix in fixtures:
+                fix_object = fix()
+                fix_object.__enter__()
+                self.used_fixtures.append(fix_object)
+        except case.SkipTest as exc:
+            # Disable the already collected tests that we now wish
+            # to skip.
+            for test in self:
+                test.run = noop
+                result.addSkip(test, str(exc))
+            result.addSkip(self, str(exc))
+        if intercept:
+            intercept_fixture = interceptor.Urllib3Interceptor(
+                intercept, host, port, prefix)
+            intercept_fixture.__enter__()
+            self.used_fixtures.append(intercept_fixture)
+
+    def stop(self):
+        """Stop fixtures when using pytest."""
+        for fix in reversed(self.used_fixtures):
+            fix.__exit__(None, None, None)
+
+    def _get_intercept(self):
+        fixtures = [fixture.GabbiFixture]
+        intercept = host = port = prefix = None
         try:
             first_test = self._find_first_full_test()
             fixtures = first_test.fixtures
@@ -62,19 +110,7 @@ class GabbiSuite(suite.TestSuite):
         except AttributeError:
             pass
 
-        try:
-            with fixture.nest([fix() for fix in fixtures]):
-                if intercept:
-                    with interceptor.Urllib3Interceptor(
-                            intercept, host, port, prefix):
-                        result = super(GabbiSuite, self).run(result, debug)
-                else:
-                    result = super(GabbiSuite, self).run(result, debug)
-        except case.SkipTest as exc:
-            for test in self._tests:
-                result.addSkip(test, str(exc))
-
-        return result
+        return fixtures, intercept, host, port, prefix
 
     def _find_first_full_test(self):
         """Traverse a sparse test suite to find the first HTTPTestCase.

gabbi/tests/test_gabbits_pytest.py

@@ -28,6 +28,7 @@ TESTS_DIR = 'gabbits_intercept'
 
 def test_from_build():
 
+    os.environ['GABBI_TEST_URL'] = 'takingnames'
     test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
     test_generator = driver.py_test_generator(
         test_dir, intercept=simple_wsgi.SimpleWsgi,

test-failskip.sh

@@ -5,9 +5,13 @@
 GREP_FAIL_MATCH='expected failures=11,'
 GREP_SKIP_MATCH='skipped=2,'
 GREP_UXSUC_MATCH='unexpected successes=1'
+PYTEST_MATCH='2 skipped, 11 xfailed'
 
 python setup.py testr && \
     for match in "${GREP_FAIL_MATCH}" "${GREP_UXSUC_MATCH}" "${GREP_SKIP_MATCH}"; do
         testr last --subunit | subunit2pyunit 2>&1 | \
             grep "${match}"
     done
+
+# Make sure pytest failskips too
+py.test gabbi | grep "$PYTEST_MATCH"