Add the concept of an inner fixture

An inner fixture is a fixture that runs per test, rather than
per test file. As implemented these fixtures are of the class
fixtures.Fixture from the python package 'fixtures'[1].

The way in which these are useful is for capturing per-test
output (stdout, stderr, logs and the like) or otherwise performing
setUp and cleanUp before and after an individual test. In many
cases this is not something that is needed in a simple gabbi run
but in some contexts (for example those with many tests in a broadly
concurrent environment) it can be important to chunkify any stray
output that the tests produce and lump it with an individual test in
a way that doesn't get interleaved with multiple streams of output.

By default nothing changes, but if someone uses the inner_fixtures
argument to build_tests and passes some fixtures (the docs will be
updated in later commits) they will be used.

This is a first commit that is not fully working but demonstrates
the concept. The tests are not working well because of global
state in the response handlers messing with this. Will fix that
elsewhere.

.testr.conf

@@ -2,4 +2,4 @@
 test_command=${PYTHON:-python} -m subunit.run discover gabbi $LISTOPT $IDOPTION
 test_id_option=--load-list $IDFILE
 test_list_option=--list
-group_regex=gabbi\.suitemaker\.(test_[^_]+_[^_]+)
+group_regex=(?:gabbi\.suitemaker\.(test_[^_]+_[^_]+)|tests\.test_(?:intercept|inner_fixture)\.([^_]+))

gabbi/case.py

@@ -25,10 +25,10 @@ import os
 import re
 import sys
 import time
-import unittest
 from unittest import case
 from unittest import result
 
+import fixtures
 import six
 from six.moves import http_cookies
 from six.moves.urllib import parse as urlparse
@@ -94,7 +94,7 @@ def potentialFailure(func):
     return wrapper
 
 
-class HTTPTestCase(unittest.TestCase):
+class HTTPTestCase(fixtures.TestWithFixtures):
     """Encapsulate a single HTTP request as a TestCase.
 
     If the test is a member of a sequence of requests, ensure that prior
@@ -110,6 +110,8 @@ class HTTPTestCase(unittest.TestCase):
     def setUp(self):
         if not self.has_run:
             super(HTTPTestCase, self).setUp()
+            for fixture in self.inner_fixtures:
+                self.useFixture(fixture())
 
     def tearDown(self):
         if not self.has_run:

gabbi/driver.py

@@ -42,7 +42,7 @@ from gabbi import utils
 def build_tests(path, loader, host=None, port=8001, intercept=None,
                 test_loader_name=None, fixture_module=None,
                 response_handlers=None, prefix='', require_ssl=False,
-                url=None):
+                url=None, inner_fixtures=None):
     """Read YAML files from a directory to create tests.
 
     Each YAML file represents an ordered sequence of HTTP requests.
@@ -60,6 +60,8 @@ def build_tests(path, loader, host=None, port=8001, intercept=None,
     :param prefix: A URL prefix for all URLs that are not fully qualified.
     :param url: A full URL to test against. Replaces host, port and prefix.
     :param require_ssl: If ``True``, make all tests default to using SSL.
+    :param inner_fixtures: A list of ``Fixtures`` to use per test
+    :type inner_fixtures: List of fixtures.Fixture clases.
     :rtype: TestSuite containing multiple TestSuites (one for each YAML file).
     """
 
@@ -109,7 +111,8 @@ def build_tests(path, loader, host=None, port=8001, intercept=None,
 
         file_suite = suitemaker.test_suite_from_dict(
             loader, test_base_name, suite_dict, path, host, port,
-            fixture_module, intercept, prefix, test_loader_name)
+            fixture_module, intercept, prefix, test_loader_name,
+            inner_fixtures)
         top_suite.addTest(file_suite)
     return top_suite
 

gabbi/suitemaker.py

@@ -37,7 +37,7 @@ class TestMaker(object):
 
     def __init__(self, test_base_name, test_defaults, test_directory,
                  fixture_classes, loader, host, port, intercept, prefix,
-                 test_loader_name=None):
+                 test_loader_name=None, inner_fixtures=None):
         self.test_base_name = test_base_name
         self.test_defaults = test_defaults
         self.default_keys = set(test_defaults.keys())
@@ -49,6 +49,7 @@ class TestMaker(object):
         self.intercept = intercept
         self.prefix = prefix
         self.test_loader_name = test_loader_name
+        self.inner_fixtures = inner_fixtures or []
 
     def make_one_test(self, test_dict, prior_test):
         """Create one single HTTPTestCase.
@@ -83,6 +84,7 @@ class TestMaker(object):
                             {'test_data': test,
                              'test_directory': self.test_directory,
                              'fixtures': self.fixture_classes,
+                             'inner_fixtures': self.inner_fixtures,
                              'http': http_class,
                              'host': self.host,
                              'intercept': self.intercept,
@@ -164,7 +166,7 @@ class TestBuilder(type):
 
 def test_suite_from_dict(loader, test_base_name, suite_dict, test_directory,
                          host, port, fixture_module, intercept, prefix='',
-                         test_loader_name=None):
+                         test_loader_name=None, inner_fixtures=None):
     """Generate a GabbiSuite from a dict represent a list of tests.
 
     The dict takes the form:
@@ -200,7 +202,7 @@ def test_suite_from_dict(loader, test_base_name, suite_dict, test_directory,
 
     test_maker = TestMaker(test_base_name, default_test_dict, test_directory,
                            fixture_classes, loader, host, port, intercept,
-                           prefix, test_loader_name)
+                           prefix, test_loader_name, inner_fixtures)
     file_suite = suite.GabbiSuite()
     prior_test = None
     for test_dict in test_data:

gabbi/tests/gabbits_inner/inner.yaml

@@ -0,0 +1,14 @@
+
+fixtures:
+    - OuterFixture
+
+tests:
+
+- name: get one
+  GET: /
+
+- name: get two
+  GET: /
+
+- name: get three
+  GET: /

gabbi/tests/test_inner_fixture.py

@@ -0,0 +1,72 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+"""Test the works of inner and outer fixtures.
+
+An "outer" fixture runs one per test suite. An "inner is per test.
+"""
+
+import copy
+import os
+import sys
+
+import fixtures
+
+from gabbi import case
+from gabbi import driver
+from gabbi import fixture
+from gabbi import handlers
+from gabbi.tests import simple_wsgi
+
+
+TESTS_DIR = 'gabbits_inner'
+COUNT_INNER = 0
+COUNT_OUTER = 0
+
+
+class OuterFixture(fixture.GabbiFixture):
+    """Assert an outer fixture is only started once and is stopped."""
+
+    def start_fixture(self):
+        # TODO(cdent): Work around a scoping bug in response
+        # handlers (the class variable has had a test handler
+        # appended to it). The content-handlers branch fixes this so we
+        # should just switch to that.
+        case.HTTPTestCase.response_handlers = handlers.RESPONSE_HANDLERS
+        case.HTTPTestCase.base_test = copy.copy(case.BASE_TEST)
+        global COUNT_OUTER
+        COUNT_OUTER += 1
+
+    def stop_fixture(self):
+        assert COUNT_OUTER == 1
+
+
+class InnerFixture(fixtures.Fixture):
+    """Test that setUp is called 3 times."""
+
+    def setUp(self):
+        super(InnerFixture, self).setUp()
+        global COUNT_INNER
+        COUNT_INNER += 1
+
+    def cleanUp(self):
+        super(InnerFixture, self).cleanUp()
+        assert 1 <= COUNT_INNER <= 3
+
+
+def load_tests(loader, tests, pattern):
+    test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
+    return driver.build_tests(test_dir, loader, host=None,
+                              intercept=simple_wsgi.SimpleWsgi,
+                              fixture_module=sys.modules[__name__],
+                              inner_fixtures=[InnerFixture],
+                              test_loader_name=__name__)

gabbi/tests/test_runner.py

@@ -51,6 +51,9 @@ class RunnerTest(unittest.TestCase):
 
         self._argv = sys.argv
         sys.argv = ['gabbi-run', '%s:%s' % (host, port)]
+        # Cleanup the custom response_handler
+        case.HTTPTestCase.response_handlers = []
+        case.HTTPTestCase.base_test = copy.copy(case.BASE_TEST)
 
     def tearDown(self):
         sys.stdin = self._stdin

gabbi/tests/test_syntax_warning.py

@@ -12,12 +12,15 @@
 # under the License.
 """Test that the driver warns on bad yaml name."""
 
+import copy
 import os
 import unittest
 import warnings
 
+from gabbi import case
 from gabbi import driver
 from gabbi import exception
+from gabbi import handlers
 
 
 TESTS_DIR = 'warning_gabbits'
@@ -29,6 +32,12 @@ class DriverTest(unittest.TestCase):
         super(DriverTest, self).setUp()
         self.loader = unittest.defaultTestLoader
         self.test_dir = os.path.join(os.path.dirname(__file__), TESTS_DIR)
+        # TODO(cdent): Work around a scoping bug in response
+        # handlers (the class variable has had a test handler
+        # appended to it). The content-handlers branch fixes this so we
+        # should just switch to that.
+        case.HTTPTestCase.response_handlers = handlers.RESPONSE_HANDLERS
+        case.HTTPTestCase.base_test = copy.copy(case.BASE_TEST)
 
     def test_driver_warngs_on_files(self):
         with warnings.catch_warnings(record=True) as the_warnings:

requirements.txt

@@ -6,3 +6,4 @@ urllib3>=1.11.0
 jsonpath-rw-ext>=1.0.0
 wsgi-intercept>=1.2.2
 colorama
+fixtures